From 6bd344a14a86ea11e5fe33b1cff5be2ec46dc599 Mon Sep 17 00:00:00 2001 From: Karan Shah Date: Wed, 19 Feb 2025 08:51:57 +0530 Subject: [PATCH] Move out base image build from dockerize (#1383) * Assume base image available in dockerize Signed-off-by: Shah, Karan * Localise base image build Signed-off-by: Shah, Karan * Update docstring Signed-off-by: Shah, Karan * Update docstring Signed-off-by: Shah, Karan --------- Signed-off-by: Shah, Karan --- .dockerignore | 2 +- .../workflows/tr_docker_gramine_direct.yml | 6 +- .github/workflows/tr_docker_native.yml | 6 +- docs/installation.md | 72 ----------------- docs/installation.rst | 80 +++++++++++++++++++ openfl-docker/Dockerfile.base | 6 +- openfl/interface/workspace.py | 41 ++++------ scripts/build_base_image.sh | 14 ---- 8 files changed, 108 insertions(+), 119 deletions(-) delete mode 100644 docs/installation.md create mode 100644 docs/installation.rst delete mode 100755 scripts/build_base_image.sh diff --git a/.dockerignore b/.dockerignore index d5483e26dc..790526f754 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,3 @@ -.git *.egg-info build +venv \ No newline at end of file diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml index 286e4d1492..7d19c150e9 100644 --- a/.github/workflows/tr_docker_gramine_direct.yml +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -25,6 +25,10 @@ jobs: python -m pip install --upgrade pip pip install . + - name: Build base image + run: | + docker build -t openfl -f openfl-docker/Dockerfile.base . + - name: Create workspace image run: | fx workspace create --prefix example_workspace --template keras/mnist @@ -35,7 +39,7 @@ jobs: # https://github.com/gramineproject/examples/issues/33 sed -i 's/write_logs: true/write_logs: false/g' plan/plan.yaml - fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }} + fx workspace dockerize --base-image openfl --save - name: Create certificate authority for workspace run: | diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml index c1aa892787..167106076f 100644 --- a/.github/workflows/tr_docker_native.yml +++ b/.github/workflows/tr_docker_native.yml @@ -25,12 +25,16 @@ jobs: python -m pip install --upgrade pip pip install . + - name: Build base image + run: | + docker build -t openfl -f openfl-docker/Dockerfile.base . + - name: Create workspace image run: | fx workspace create --prefix example_workspace --template keras/mnist cd example_workspace fx plan initialize -a localhost - fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }} + fx workspace dockerize --base-image openfl --save - name: Create certificate authority for workspace run: | diff --git a/docs/installation.md b/docs/installation.md deleted file mode 100644 index 2347abc70e..0000000000 --- a/docs/installation.md +++ /dev/null @@ -1,72 +0,0 @@ -# Installation - -This document provides instructions for installing OpenFL; either in a Python virtual environment or as a docker container. - -## Using `pip` - -We recommend using a Python virtual environment. Refer to the [venv installation guide](https://docs.python.org/3/library/venv.html) for details. - -* From PyPI (latest stable release): - - ```bash - pip install openfl - ``` - -* For development (editable build): - - ```bash - git clone https://github.com/securefederatedai/openfl.git && cd openfl - pip install -e . - ``` - -* Nightly (from the tip of `develop` branch): - - ```bash - pip install git+https://github.com/securefederatedai/openfl.git@develop - ``` - -Verify installation using the `fx --help` command. - -```bash -OpenFL - Open Federated Learning - -BASH COMPLETE ACTIVATION - -Run in terminal: -_FX_COMPLETE=bash_source fx > ~/.fx-autocomplete.sh -source ~/.fx-autocomplete.sh -If ~/.fx-autocomplete.sh already exists: -source ~/.fx-autocomplete.sh - -CORRECT USAGE - -fx [options] [command] [subcommand] [args] - -GLOBAL OPTIONS - --l, --log-level TEXT Logging verbosity level. ---no-warnings Disable third-party warnings. ---help Show this message and exit. - -AVAILABLE COMMANDS -... -``` - -## Using `docker` - -This method can be used to run federated learning experiments in an isolated environment. Install and verify installation of Docker engine on all nodes in the federation. Refer to the Docker installation [guide](https://docs.docker.com/engine/install/) for details. - -* Pull the latest image: - - ```bash - docker pull ghcr.io/securefederatedai/openfl:latest - ``` - -* Build from source: - ```bash - git clone https://github.com/securefederatedai/openfl.git && cd openfl - git checkout develop - ``` - ```bash - ./scripts/build_base_image.sh - ``` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000000..2f79e2c13d --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,80 @@ +Installation +============ + +This document provides instructions for installing OpenFL; either in a Python virtual environment or as a docker container. + +Using ``pip`` +----------- + +We recommend using a Python virtual environment. Refer to the `venv installation guide `_ for details. + +* From PyPI (latest stable release): + + .. code-block:: bash + + pip install openfl + +* For development (editable build): + + .. code-block:: bash + + git clone https://github.com/securefederatedai/openfl.git && cd openfl + pip install -e . + +* Nightly (from the tip of `develop` branch): + + .. code-block:: bash + + pip install git+https://github.com/securefederatedai/openfl.git@develop + +Verify installation using the ``fx --help`` command. + +.. code-block:: bash + + OpenFL - Open Federated Learning + + BASH COMPLETE ACTIVATION + + Run in terminal: + _FX_COMPLETE=bash_source fx > ~/.fx-autocomplete.sh + source ~/.fx-autocomplete.sh + If ~/.fx-autocomplete.sh already exists: + source ~/.fx-autocomplete.sh + + CORRECT USAGE + + fx [options] [command] [subcommand] [args] + + GLOBAL OPTIONS + + -l, --log-level TEXT Logging verbosity level. + --no-warnings Disable third-party warnings. + --help Show this message and exit. + + AVAILABLE COMMANDS + ... + +Using ``docker`` +-------------- + +This method can be used to run federated learning experiments in an isolated environment. Install and verify installation of Docker engine on all nodes in the federation. Refer to the Docker installation `guide `_ for details. + +* Pull the latest image: + + .. code-block:: bash + + docker pull ghcr.io/securefederatedai/openfl:latest + +* Build from source: + + .. code-block:: bash + + git clone https://github.com/securefederatedai/openfl.git && cd openfl + git checkout develop + + .. code-block:: bash + + docker build -t openfl -f openfl-docker/Dockerfile.base . + + .. note:: + This command copies current context (i.e. OpenFL root directory) to the base image. Ensure that the ``.dockerignore`` file is configured to exclude unnecessary files and directories (like secrets or local virtual environments). \ No newline at end of file diff --git a/openfl-docker/Dockerfile.base b/openfl-docker/Dockerfile.base index 1de3130217..424b249ca4 100644 --- a/openfl-docker/Dockerfile.base +++ b/openfl-docker/Dockerfile.base @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # ------------------------------------ # OpenFL Base Image w/ Gramine support -# $> docker build . -t openfl -f Dockerfile.base [--build-arg OPENFL_REVISION=GIT_URL@COMMIT_ID] # ------------------------------------ FROM ubuntu:22.04 AS base @@ -43,8 +42,9 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ rm -rf /var/lib/apt/lists/* # Install OpenFL. -ARG OPENFL_REVISION=https://github.com/securefederatedai/openfl.git@develop -RUN pip install --no-cache-dir git+${OPENFL_REVISION} && \ +WORKDIR /repo +COPY . . +RUN pip install --no-cache-dir . && \ INSTALL_SOURCES=yes /opt/venv/lib/python3.10/site-packages/openfl-docker/licenses.sh # Create an unprivileged user. diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index 42ed8d7983..071701fbc9 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -385,14 +385,14 @@ def export_() -> str: "--save", is_flag=True, default=False, - help="Export the docker image as ``.tar`` file.", + help="Export the workspace image as ``.tar`` file.", show_default=True, ) @option( "--rebuild", is_flag=True, default=False, - help="If set, rebuilds docker images with ``--no-cache`` option.", + help="If set, rebuilds workspace image with ``--no-cache`` option.", show_default=True, ) @option( @@ -407,46 +407,33 @@ def export_() -> str: ), ) @option( - "--revision", + "--base-image", + "base_image", required=False, - default=None, + default="ghcr.io/securefederatedai/openfl:latest", help=( - "Optional, version of OpenFL source code to build base image from. " - "If unspecified, default value in `Dockerfile.base` will be used, " - "typically the latest stable release. " - "Format: ``OPENFL_GIT_URL@``" + "OpenFL base image to use for creating a workspace. " + "If unspecified, default latest base image will be pulled. " + "To build a base image from source, follow docker build " + "`instructions. `_ " + "Format: ``IMAGE_URI:TAG``" ), + show_default=True, ) @pass_context -def dockerize_(context, save: bool, rebuild: bool, enclave_key: str, revision: str): +def dockerize_(context, save: bool, rebuild: bool, enclave_key: str, base_image: str): """Packages current workspace as a TEE-ready docker image.""" - # Docker build options - options = [] - options.append("--no-cache" if rebuild else "") - options.append(f"--build-arg OPENFL_REVISION={revision}" if revision else "") - options = " ".join(options) - # Export workspace archive = context.invoke(export_) workspace_name, _ = archive.split(".") - # Build OpenFL base image. - logging.info("Building OpenFL Base image") - base_image_build_cmd = ( - "DOCKER_BUILDKIT=1 docker build {options} -t {image_name} -f {dockerfile} {build_context}" - ).format( - options=options, - image_name="openfl", - dockerfile=os.path.join(SITEPACKS, "openfl-docker", "Dockerfile.base"), - build_context=".", - ) - _execute(base_image_build_cmd) - # Build workspace image. options = [] options.append("--no-cache" if rebuild else "") + options.append(f"--build-arg BASE_IMAGE={base_image}") options = " ".join(options) + logging.info(f"Using base image: {base_image}") if enclave_key is None: _execute("openssl genrsa -out key.pem -3 3072") enclave_key = os.path.abspath("key.pem") diff --git a/scripts/build_base_image.sh b/scripts/build_base_image.sh deleted file mode 100755 index 0d4d42b6bb..0000000000 --- a/scripts/build_base_image.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -e - -TAG=${1:-'openfl'} -REPO=https://github.com/securefederatedai/openfl.git -REVISION=develop -echo "Using OpenFL: ${REPO}@${REVISION}" - -docker build \ --t ${TAG} \ ---build-arg OPENFL_REVISION=${REPO}@${REVISION} \ --f openfl-docker/Dockerfile.base . - -echo "Successfully built OpenFL base docker image: ${TAG}"