Skip to content

Commit

Permalink
Pytorch notebook/init (#6)
Browse files Browse the repository at this point in the history
* fix: removed multiplatform build in CI
  • Loading branch information
eroan-marie authored Dec 4, 2024
1 parent b74fee2 commit 33dfad1
Show file tree
Hide file tree
Showing 33 changed files with 6,237 additions and 2,676 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/Build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ jobs:
uses: docker/build-push-action@v6
with:
context: ${{ env.IMAGE }}
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
push: true
tags: |
${{env.DOCKER_ORG}}/${{ env.IMAGE }}:master
Expand Down Expand Up @@ -98,7 +98,7 @@ jobs:
with:
context: ${{ matrix.IMAGE }}
push: true
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
tags: |
${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:master
${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:${{env.DOCKER_TAG}}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/CondaLock.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ jobs:
run: |
cd ${{ matrix.IMAGE }}
if [ ${{ matrix.IMAGE }} = "base-notebook" ]; then
conda-lock lock -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64
conda-lock lock -p linux-64
elif [ ${{ matrix.IMAGE }} = "pangeo-notebook" ]; then
conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64
conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64
else
# Linux-64 ONLY
conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/Test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
uses: docker/build-push-action@v6
with:
context: base-image
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
tags: localhost:5000/cnes/base-image:PR
push: true
cache-from: type=gha
Expand All @@ -64,7 +64,7 @@ jobs:
uses: docker/build-push-action@v6
with:
context: ${{ matrix.IMAGE }}
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
push: true
tags: localhost:5000/${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:PR
build-args: PANGEO_BASE_IMAGE_TAG=PR
Expand Down
10 changes: 6 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# Makefile for convenience, (doesn't look for command outputs)
.PHONY: all
all: base-image base-notebook pangeo-notebook ml-notebook pytorch-notebook
all: base-image base-notebook pangeo-notebook pytorch-notebook
TESTDIR=/srv/test

.PHONY: base-image
Expand All @@ -13,7 +13,7 @@ base-image :
.PHONY: base-notebook
base-notebook : base-image
cd base-notebook ; \
conda-lock lock --mamba -f environment.yml -p linux-64; \
conda-lock lock -f environment.yml -p linux-64; \
conda-lock render -k explicit -p linux-64; \
../generate-packages-list.py conda-linux-64.lock > packages.txt; \
docker build -t cnes/base-notebook:master . --no-cache --progress=plain --platform linux/amd64; \
Expand All @@ -23,7 +23,7 @@ base-notebook : base-image
pangeo-notebook : base-image
cd pangeo-notebook ; \
cp -r ../base-notebook/resources . ; \
conda-lock lock --mamba -f environment.yml -f ../base-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
conda-lock render -k explicit -p linux-64; \
../generate-packages-list.py conda-linux-64.lock > packages.txt; \
../merge-apt.sh ../base-notebook/apt.txt apt.txt; \
Expand All @@ -33,8 +33,10 @@ pangeo-notebook : base-image
.PHONY: pytorch-notebook
pytorch-notebook : base-image
cd pytorch-notebook ; \
conda-lock lock --mamba -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
cp -r ../pangeo-notebook/resources ../base-notebook/resources . ; \
conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
conda-lock render -k explicit -p linux-64; \
../generate-packages-list.py conda-linux-64.lock > packages.txt; \
../merge-apt.sh ../pangeo-notebook/apt.txt ../base-notebook/apt.txt apt.txt; \
docker build -t cnes/pytorch-notebook:master . ; \
docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) cnes/pytorch-notebook:master ./run_tests.sh pytorch-notebook
2 changes: 1 addition & 1 deletion base-image/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ ONBUILD RUN echo "Checking for 'jupyter_notebook_config.py'..." \
&& cp jupyter_notebook_config.py /etc/jupyter \
; fi

ONBUILD USER ${NB_USER}
# ONBUILD USER ${NB_USER}

# We want to keep our images as reproducible as possible. If a lock
# file with exact versions of all required packages is present, we use
Expand Down
2 changes: 2 additions & 0 deletions pangeo-notebook/conda-linux-64.lock
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# Generated by conda-lock.
# platform: linux-64
# input_hash: 51a7de05b0fe1efe11c5457b666b4c69da061c0c7be4c21639f5b2bb83ec843a
# input_hash: 51a7de05b0fe1efe11c5457b666b4c69da061c0c7be4c21639f5b2bb83ec843a
@EXPLICIT
https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
https://conda.anaconda.org/conda-forge/linux-64/_py-xgboost-mutex-2.0-gpu_0.tar.bz2#7702188077361f43a4d61e64c694f850
Expand Down Expand Up @@ -439,6 +440,7 @@ https://conda.anaconda.org/conda-forge/noarch/traittypes-0.2.1-pyh9f0ad1d_2.tar.
https://conda.anaconda.org/conda-forge/noarch/trollsift-0.5.1-pyhd8ed1ab_0.conda#5819998373586657ba9868756c0aaf8a
https://conda.anaconda.org/conda-forge/noarch/typer-slim-0.13.1-pyhff2d567_0.conda#85283fb942fa2604c3db03483027ced2
https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.12.2-hd8ed1ab_0.conda#52d648bd608f5737b123f510bb5514b5
https://conda.anaconda.org/conda-forge/linux-64/ucx-1.17.0-h0104b51_3.conda#cbd9d35cbfb829275fde06d292afcc8b
https://conda.anaconda.org/conda-forge/noarch/urllib3-1.26.19-pyhd8ed1ab_0.conda#6bb37c314b3cc1515dcf086ffe01c46e
https://conda.anaconda.org/conda-forge/noarch/webob-1.8.9-pyhd8ed1ab_0.conda#ff98f23ad74d2a3256debcd9df65d37d
https://conda.anaconda.org/conda-forge/noarch/yamale-5.2.1-pyhca7485f_0.conda#c089f90a086b6214c5606368d0d3bad0
Expand Down
Loading

0 comments on commit 33dfad1

Please sign in to comment.