From a21bb621c915c9984f43e2c6867c8b5e0b516027 Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Tue, 23 Apr 2024 16:33:16 +0400 Subject: [PATCH] chore: Move feast install to docker build in java it tests (#4126) * chore: Move feast install to docker build in java it tests Signed-off-by: tokoko * remove commented out lines in compose file Signed-off-by: tokoko * make local compose mode default Signed-off-by: tokoko * limit COPY contents Signed-off-by: tokoko * remove requirements.txt from java tests docker image Signed-off-by: tokoko * include pyproject.toml in dockerfile Signed-off-by: tokoko * change links to depends_on Signed-off-by: tokoko * try updating setup-python to v5 Signed-off-by: tokoko * pin macos image to macos-12 Signed-off-by: tokoko * force rerun Signed-off-by: tokoko --------- Signed-off-by: tokoko Signed-off-by: Lokesh Rangineni --- .devcontainer/devcontainer.json | 5 +---- .../fork_pr_integration_tests_aws.yml | 2 +- .../fork_pr_integration_tests_gcp.yml | 2 +- .../fork_pr_integration_tests_snowflake.yml | 2 +- .github/workflows/build_wheels.yml | 6 +++--- .github/workflows/nightly-ci.yml | 2 +- .github/workflows/unit_tests.yml | 4 ++-- .../java/feast/serving/it/ServingEnvironment.java | 7 ++----- .../docker-compose/docker-compose-redis-it.yml | 11 ++++------- .../resources/docker-compose/feast10/Dockerfile | 13 +++++++------ .../resources/docker-compose/feast10/entrypoint.sh | 4 ---- .../docker-compose/feast10/requirements.txt | 6 ------ 12 files changed, 23 insertions(+), 41 deletions(-) delete mode 100644 java/serving/src/test/resources/docker-compose/feast10/requirements.txt diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b4da25737f..e82fd04db4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,10 +7,7 @@ }, "ghcr.io/devcontainers/features/python:1": { "version": "3.9" - }, - "ghcr.io/meaningful-ooo/devcontainer-features/homebrew:2": { - "version": "latest" } }, - "postCreateCommand": "brew install mysql && pip install -e '.[dev]' && make compile-protos-python" + "postCreateCommand": "pip install -e '.[dev]' && make compile-protos-python" } diff --git a/.github/fork_workflows/fork_pr_integration_tests_aws.yml b/.github/fork_workflows/fork_pr_integration_tests_aws.yml index be75c4f987..52d7112a64 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_aws.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_aws.yml @@ -138,7 +138,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macos-12' run: | brew install apache-arrow brew install pkg-config diff --git a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml index 0793fbd6e5..337d8040ae 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_gcp.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_gcp.yml @@ -82,7 +82,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macOS-12' run: | brew install apache-arrow brew install pkg-config diff --git a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml index b9b6f8df06..a3484a3462 100644 --- a/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml +++ b/.github/fork_workflows/fork_pr_integration_tests_snowflake.yml @@ -72,7 +72,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macos-12' run: | brew install apache-arrow brew install pkg-config diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f0851f5bb0..ca0a7dcfe2 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -79,7 +79,7 @@ jobs: build-source-distribution: name: Build source distribution - runs-on: macos-latest + runs-on: macos-12 steps: - uses: actions/checkout@v3 - name: Setup Python @@ -136,7 +136,7 @@ jobs: needs: [build-python-wheel, build-source-distribution, get-version] strategy: matrix: - os: [ubuntu-latest, macos-latest ] + os: [ubuntu-latest, macos-12 ] python-version: ["3.9", "3.10"] from-source: [ True, False ] env: @@ -165,7 +165,7 @@ jobs: name: wheels path: dist - name: Install OS X dependencies - if: matrix.os == 'macos-latest' + if: matrix.os == 'macos-12' run: brew install coreutils - name: Install wheel if: ${{ !matrix.from-source }} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 4dea41d4ad..3292d6bcb5 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -202,7 +202,7 @@ jobs: sudo apt update sudo apt install -y -V libarrow-dev - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' + if: matrix.os == 'macos-12' run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f3f91bb67f..ff7c3d5e23 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -8,9 +8,9 @@ jobs: fail-fast: false matrix: python-version: [ "3.9", "3.10" ] - os: [ ubuntu-latest, macOS-latest ] + os: [ ubuntu-latest, macos-12 ] exclude: - - os: macOS-latest + - os: macos-12 python-version: "3.9" env: OS: ${{ matrix.os }} diff --git a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java index 5489203856..356524399a 100644 --- a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java +++ b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java @@ -62,11 +62,8 @@ static void globalSetup() { .withExposedService("redis", 6379) .withExposedService( "feast", 8080, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(180))) - .withTailChildContainers(true); - - if (System.getenv("FEAST_TESTCONTAINERS_LOCAL_COMPOSE") != null) { - environment = environment.withLocalCompose(true); - } + .withTailChildContainers(true) + .withLocalCompose(true); environment.start(); } diff --git a/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml index 0522750d99..142efe7fa2 100644 --- a/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml +++ b/java/serving/src/test/resources/docker-compose/docker-compose-redis-it.yml @@ -1,5 +1,3 @@ -version: '3' - services: redis: image: redis:6.2 @@ -7,11 +5,10 @@ services: ports: - "6379" feast: - build: feast10 + build: + context: ../../../../../../ + dockerfile: java/serving/src/test/resources/docker-compose/feast10/Dockerfile ports: - "8080" - links: + depends_on: - redis - volumes: - - $PWD/../../../../../../:/mnt/feast - diff --git a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile index 7e36658cae..8b3c5b3d3d 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/Dockerfile +++ b/java/serving/src/test/resources/docker-compose/feast10/Dockerfile @@ -1,12 +1,13 @@ FROM python:3.9 -WORKDIR /usr/src/ - -COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt - WORKDIR /app -COPY . . +COPY java/serving/src/test/resources/docker-compose/feast10/ . +COPY sdk/python /mnt/feast/sdk/python +COPY protos /mnt/feast/protos +COPY setup.py /mnt/feast/setup.py +COPY pyproject.toml /mnt/feast/pyproject.toml +COPY README.md /mnt/feast/README.md +RUN cd /mnt/feast && SETUPTOOLS_SCM_PRETEND_VERSION="0.1.0" pip install .[grpcio,redis] EXPOSE 8080 CMD ["./entrypoint.sh"] diff --git a/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh b/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh index 0690b734c3..82d9399521 100755 --- a/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh +++ b/java/serving/src/test/resources/docker-compose/feast10/entrypoint.sh @@ -2,10 +2,6 @@ set -e -# feast root directory is expected to be mounted (eg, by docker compose) -cd /mnt/feast -pip install -e '.[grpcio,redis]' - cd /app python materialize.py feast serve_transformations --port 8080 diff --git a/java/serving/src/test/resources/docker-compose/feast10/requirements.txt b/java/serving/src/test/resources/docker-compose/feast10/requirements.txt deleted file mode 100644 index 6ba2c53d81..0000000000 --- a/java/serving/src/test/resources/docker-compose/feast10/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# for source generation -pyarrow==14.0.1 - -# temp fixes -proto-plus -Jinja2>=2.0.0 \ No newline at end of file