ENG-627: Docs for CloudCompute Mount Argument (#15182)

fixed conflicts

(cherry picked from commit 2041908)

docs/source-app/api_reference/storage.rst

@@ -20,6 +20,7 @@ ______________
     ~path.Path
     ~drive.Drive
     ~payload.Payload
+    ~mount.Mount
 
 ----
 
@@ -56,6 +57,14 @@ Learn more about Storage
    :height: 180
    :tag: Intermediate
 
+.. displayitem::
+   :header: The Mount Object.
+   :description: Mount an AWS S3 Bucket When Running on the Cloud.
+   :col_css: col-md-4
+   :button_link:  ../workflows/mount_aws_s3_bucket.html
+   :height: 180
+   :tag: Intermediate
+
 .. raw:: html
 
         </div>

docs/source-app/api_references.rst

@@ -72,6 +72,7 @@ _______
     ~path.Path
     ~drive.Drive
     ~payload.Payload
+    ~mount.Mount
 
 Learn more about :ref:`Storage <storage>`.
 
@@ -90,3 +91,18 @@ _______
     ~cloud.CloudRuntime
     ~singleprocess.SingleProcessRuntime
     ~multiprocess.MultiProcessRuntime
+
+----
+
+lightning_app.utilities.packaging
+_________________________________
+
+.. currentmodule:: lightning_app.utilities.packaging
+
+.. autosummary::
+    :toctree: generated/
+    :nosignatures:
+    :template: classtemplate_no_index.rst
+
+    ~cloud_compute.CloudCompute
+    ~build_config.BuildConfig

docs/source-app/glossary/index.rst

@@ -112,6 +112,13 @@ Glossary
    :button_link: ../core_api/lightning_app/index.html
    :height: 100
 
+.. displayitem::
+   :header: Mounts
+   :description: Mount Cloud Data
+   :col_css: col-md-6
+   :button_link: mount.html
+   :height: 180
+
 .. displayitem::
    :header: Sharing Components
    :description: Let's create an ecosystem altogether

docs/source-app/glossary/mount.rst

@@ -0,0 +1 @@
+.. include:: ../workflows/mount_cloud_object_store.rst

docs/source-app/workflows/index.rst

@@ -187,6 +187,14 @@ How to:
    :button_link: ssh/index.html
    :height: 180
 
+.. displayitem::
+   :header: Mount Cloud Data
+   :description: Learn how Lightning Mounts are used to make the contents of an cloud object store bucket available on disk when running in the cloud.
+   :col_css: col-md-4
+   :button_link: mount_cloud_object_store.html
+   :height: 180
+
+
 
 .. raw:: html
 

docs/source-app/workflows/mount_cloud_object_store.rst

@@ -0,0 +1,141 @@
+:orphan:
+
+##############
+Add Cloud Data
+##############
+
+**Audience:** Users who want to read files stored in a Cloud Object Bucket in an app.
+
+******************************
+Mounting Public AWS S3 Buckets
+******************************
+
+===================
+Add Mount to a Work
+===================
+
+To mount data from a cloud bucket to your app compute, initialize a :class:`~lightning_app.storage.mount.Mount`
+object with the source path of the s3 bucket and the absolute directory path where it should be mounted and
+pass the :class:`~lightning_app.storage.mount.Mount` to the :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute`
+of the :class:`~lightning_app.core.work.LightningWork` it should be mounted on.
+
+In this example, we will mount an S3 bucket: ``s3://ryft-public-sample-data/esRedditJson/`` to ``/content/esRedditJson/``.
+
+.. code-block:: python
+
+    from lightning_app import CloudCompute
+    from lightning_app.storage import Mount
+
+    self.my_work = MyWorkClass(
+        cloud_compute=CloudCompute(
+            mounts=Mount(
+                source="s3://ryft-public-sample-data/esRedditJson/",
+                mount_path="/content/esRedditJson/",
+            ),
+        )
+    )
+
+You can also pass multiple mounts to a single work by passing a ``List[Mount(...), ...]`` to the
+``CloudCompute(mounts=...)`` argument.
+
+.. note::
+
+    * Mounts supported up to 1 Million files, 5GB per file. Need larger mounts? Contact [email protected]
+    * When adding multiple mounts, each one should have a unique ``mount_path``.
+    * A maximum of 10 :class:`~lightning_app.storage.mount.Mount`\s can be added to a :class:`~lightning_app.core.work.LightningWork`.
+
+=======================
+Read Files From a Mount
+=======================
+
+Once a :class:`~lightning_app.storage.mount.Mount` object is passed to :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute`,
+you can access, list, or read any file from the mount under the specified ``mount_path``, just like you would if it
+was on your local machine.
+
+Assuming your ``mount_path`` is ``"/content/esRedditJson/"`` you can do the following:
+
+----------
+Read Files
+----------
+
+.. code-block:: python
+
+    with open("/content/esRedditJson/esRedditJson1", "r") as f:
+        some_data = f.read()
+
+    # do something with "some_data"...
+
+----------
+List Files
+----------
+
+.. code-block:: python
+
+    files = os.listdir("/content/esRedditJson/")
+
+--------------------
+See the Full Example
+--------------------
+
+.. code-block:: python
+    :emphasize-lines: 10,15
+
+    import os
+
+    import lightning as L
+    from lightning_app import CloudCompute
+    from lightning_app.storage import Mount
+
+    class ReadMount(L.LightningWork):
+       def run(self):
+           # Print a list of files stored in the mounted S3 Bucket.
+           files = os.listdir("/content/esRedditJson/")
+           for file in files:
+               print(file)
+
+           # Read the contents of a particular file in the bucket "esRedditJson1"
+           with open("/content/esRedditJson/esRedditJson1", "r") as f:
+               some_data = f.read()
+               # do something with "some_data"...
+
+    class Flow(L.LightningFlow):
+       def __init__(self):
+           super().__init__()
+           self.my_work = ReadMount(
+               cloud_compute=CloudCompute(
+                   mounts=Mount(
+                       source="s3://ryft-public-sample-data/esRedditJson/",
+                       mount_path="/content/esRedditJson/",
+                   ),
+               )
+           )
+
+       def run(self):
+           self.my_work.run()
+
+.. note::
+
+    When running a Lighting App on your local machine, any :class:`~lightning_app.utilities.packaging.cloud_compute.CloudCompute`
+    configuration (including a :class:`~lightning_app.storage.mount.Mount`) is ignored at runtime. If you need access to
+    these files on your local disk, you should download a copy of them to your machine.
+
+.. note::
+
+    Mounted files from an S3 bucket are ``read-only``. Any modifications, additions, or deletions
+    to files in the mounted directory will not be reflected in the cloud object store.
+
+----
+
+**********************************************
+Mounting Private AWS S3 Buckets - Coming Soon!
+**********************************************
+
+We'll Let you know when this feature is ready!
+
+----
+
+************************************************
+Mounting Google Cloud GCS Buckets - Coming Soon!
+************************************************
+
+We'll Let you know when this feature is ready!

examples/app_mount/app.py

@@ -32,4 +32,4 @@ def run(self):
         self.work_1.run()
 
 
-app = L.LightningApp(Flow(), log_level="debug")
+app = L.LightningApp(Flow())