Base notebook/init (#1)

* feat: add remote desktop, nbproxy and vscode +  adapt CI/CD for our needs

.github/workflows/Build.yml

@@ -12,7 +12,7 @@ on:
       - 'update_lockfile.sh'
 
 env:
-  DOCKER_ORG: pangeo
+  DOCKER_ORG: cnes
   GITHUB_SHA: ${{ github.sha }}
   GITHUB_REF: ${{ github.ref }}
 
@@ -44,13 +44,6 @@ jobs:
         username: ${{ secrets.DOCKERHUB_USER }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-    - name: Login to Quay.io
-      uses: docker/login-action@v3
-      with:
-        registry: quay.io
-        username: ${{ secrets.QUAY_USER }}
-        password: ${{ secrets.QUAY_TOKEN }}
-
     - name: Build and push
       uses: docker/build-push-action@v6
       with:
@@ -60,15 +53,13 @@ jobs:
         tags: |
           ${{env.DOCKER_ORG}}/${{ env.IMAGE }}:master
           ${{env.DOCKER_ORG}}/${{ env.IMAGE }}:${{env.DOCKER_TAG}}
-          quay.io/${{env.DOCKER_ORG}}/${{ env.IMAGE }}:master
-          quay.io/${{env.DOCKER_ORG}}/${{ env.IMAGE }}:${{env.DOCKER_TAG}}
 
   matrix-build:
     needs: base-image
     strategy:
       fail-fast: false
       matrix:
-        IMAGE: [base-notebook, pangeo-notebook, ml-notebook, pytorch-notebook]
+        IMAGE: [base-notebook, pangeo-notebook, pytorch-notebook]
     name: ${{ matrix.IMAGE }}
     runs-on: ubuntu-latest
     steps:
@@ -102,13 +93,6 @@ jobs:
         username: ${{ secrets.DOCKERHUB_USER }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-    - name: Login to Quay.io
-      uses: docker/login-action@v3
-      with:
-        registry: quay.io
-        username: ${{ secrets.QUAY_USER }}
-        password: ${{ secrets.QUAY_TOKEN }}
-
     - name: Build and push
       uses: docker/build-push-action@v6
       with:
@@ -118,8 +102,6 @@ jobs:
         tags: |
           ${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:master
           ${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:${{env.DOCKER_TAG}}
-          quay.io/${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:master
-          quay.io/${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:${{env.DOCKER_TAG}}
 
     - name: Inspect Image
       run: |

.github/workflows/ChatOpsDispatcher.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Slash Command Dispatch
         uses: peter-evans/slash-command-dispatch@v4
         with:
-          token: ${{ secrets.PANGEOBOT_TOKEN }}
+          token: ${{ secrets.DATALABSBOT_TOKEN }}
           reaction-token: ${{ secrets.GITHUB_TOKEN }}
           config: >
             [

.github/workflows/CondaLock.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        IMAGE: [base-notebook, pangeo-notebook, ml-notebook, pytorch-notebook]
+        IMAGE: [base-notebook, pangeo-notebook, pytorch-notebook]
     name: ${{ matrix.IMAGE }}
     runs-on: ubuntu-latest
     timeout-minutes: 180
@@ -23,7 +23,7 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
         with:
-          token: ${{ secrets.PANGEOBOT_TOKEN }}
+          token: ${{ secrets.DATALABSBOT_TOKEN }}
           # These lines are critical, otherwise Pangeo-bot pushes changes directly to master from PRs!
           repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
           ref: ${{ github.event.client_payload.pull_request.head.ref }}
@@ -64,7 +64,7 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
         with:
-          token: ${{ secrets.PANGEOBOT_TOKEN }}
+          token: ${{ secrets.DATALABSBOT_TOKEN }}
           # These lines are critical, otherwise Pangeo-bot pushes changes directly to master from PRs!
           repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }}
           ref: ${{ github.event.client_payload.pull_request.head.ref }}
@@ -74,8 +74,8 @@ jobs:
 
       - name: Commit condalock files to PR
         run: |
-          git config --global user.name 'pangeo-bot'
-          git config --global user.email 'pangeo[email protected]'
+          git config --global user.name 'datalabs-bot'
+          git config --global user.email 'datalabs[email protected]'
           git status
           git add */conda*.lock */conda-lock.yml */packages.txt
           git commit -m "[condalock-command] autogenerated conda-lock files"

.github/workflows/PR.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   DeployPangeoBot:
     runs-on: ubuntu-latest
-    if: github.repository == 'pangeo-data/pangeo-docker-images'
+    if: github.repository == 'CNES/datalabs-docker-images'
 
     steps:
       - name: Checkout Repository
@@ -17,7 +17,7 @@ jobs:
       - name: Add Condalock Comment
         uses: peter-evans/create-or-update-comment@v4
         with:
-           token: ${{ secrets.PANGEOBOT_TOKEN }}
+           token: ${{ secrets.DATALABSBOT_TOKEN }}
            issue-number: ${{github.event.number}}
            body: |
              /condalock

.github/workflows/Publish.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        IMAGE: [base-image, base-notebook, pangeo-notebook, ml-notebook, pytorch-notebook]
+        IMAGE: [base-image, base-notebook, pangeo-notebook, pytorch-notebook]
     name: ${{ matrix.IMAGE }}
     runs-on: ubuntu-latest
     steps:
@@ -33,21 +33,21 @@ jobs:
         username: ${{ secrets.DOCKERHUB_USER }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-    - name: Login to Quay.io
-      uses: docker/login-action@v3
-      with:
-        registry: quay.io
-        username: ${{ secrets.QUAY_USER }}
-        password: ${{ secrets.QUAY_TOKEN }}
+    # - name: Login to Quay.io
+    #   uses: docker/login-action@v3
+    #   with:
+    #     registry: quay.io
+    #     username: ${{ secrets.QUAY_USER }}
+    #     password: ${{ secrets.QUAY_TOKEN }}
 
     - name: Create New DockerHub Tags
       run: |
         docker buildx imagetools create ${DOCKER_ORG}/${{ matrix.IMAGE }}:master \
           -t ${DOCKER_ORG}/${{ matrix.IMAGE }}:latest \
           -t ${DOCKER_ORG}/${{ matrix.IMAGE }}:${TAG}
 
-    - name: Create New Quay.io Tags
-      run: |
-        docker buildx imagetools create quay.io/${DOCKER_ORG}/${{ matrix.IMAGE }}:master \
-          -t quay.io/${DOCKER_ORG}/${{ matrix.IMAGE }}:latest \
-          -t quay.io/${DOCKER_ORG}/${{ matrix.IMAGE }}:${TAG}
+    # - name: Create New Quay.io Tags
+    #   run: |
+    #     docker buildx imagetools create quay.io/${DOCKER_ORG}/${{ matrix.IMAGE }}:master \
+    #       -t quay.io/${DOCKER_ORG}/${{ matrix.IMAGE }}:latest \
+    #       -t quay.io/${DOCKER_ORG}/${{ matrix.IMAGE }}:${TAG}

.github/workflows/Test.yml

@@ -10,7 +10,7 @@ on:
     types: [opened, synchronize, reopened]
 
 env:
-  DOCKER_ORG: pangeo
+  DOCKER_ORG: cnes
   GITHUB_SHA: ${{ github.sha }}
   GITHUB_REF: ${{ github.ref }}
 
@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        IMAGE: [base-notebook, pangeo-notebook, ml-notebook, pytorch-notebook]
+        IMAGE: [base-notebook, pangeo-notebook, pytorch-notebook]
     name: ${{ matrix.IMAGE }}
     runs-on: ubuntu-latest
     services:
@@ -69,7 +69,7 @@ jobs:
         tags: localhost:5000/${{env.DOCKER_ORG}}/${{ matrix.IMAGE }}:PR
         build-args: PANGEO_BASE_IMAGE_TAG=PR
         build-contexts: |
-          pangeo/base-image:PR=docker-image://localhost:5000/pangeo/base-image:PR
+          cnes/base-image:PR=docker-image://localhost:5000/cnes/base-image:PR
 
     - name: List Conda Packages
       run: |

.github/workflows/WatchCondaForge.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   check-version:
     runs-on: ubuntu-latest
-    if: github.repository == 'pangeo-data/pangeo-docker-images'
+    if: github.repository == 'CNES/datalabs-docker-images'
 
     steps:
       - name: Checkout Repository
@@ -57,11 +57,11 @@ jobs:
         id: cpr
         uses: peter-evans/create-pull-request@v7
         with:
-          committer: Pangeo Bot <pangeo[email protected]>
-          token: ${{ secrets.PANGEOBOT_TOKEN }}
+          committer: Datalabs Bot <datalabs[email protected]>
+          token: ${{ secrets.DATALABSBOT_TOKEN }}
           commit-message: "Update pangeo-notebook metapackage version to ${{ steps.latest_version.outputs.version }}"
           title: "Update pangeo-notebook metapackage version to ${{ steps.latest_version.outputs.version }}"
-          reviewers: "scottyhq"
+          reviewers: "eroan.marie"
           branch: ${{ env.PR_BRANCH }}
           body: |
             A new pangeo-metapackage version has been detected.

Makefile

@@ -1,3 +1,5 @@
+-include .env
+
 # Makefile for convenience, (doesn't look for command outputs)
 .PHONY: all
 all: base-image base-notebook pangeo-notebook ml-notebook pytorch-notebook
@@ -6,40 +8,32 @@ TESTDIR=/srv/test
 .PHONY: base-image
 base-image :
 	cd base-image ; \
-	docker build -t pangeo/base-image:master --progress=plain --platform linux/amd64,linux/arm64 .
+	docker build -t cnes/base-image:master --progress=plain --platform linux/amd64 .
 
 .PHONY: base-notebook
 base-notebook : base-image
 	cd base-notebook ; \
-	conda-lock lock -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64; \
+	conda-lock lock  -f environment.yml -p linux-64; \
 	conda-lock render -k explicit -p linux-64; \
 	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
-	docker build -t pangeo/base-notebook:master . --progress=plain --platform linux/amd64,linux/arm64 ; \
-	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/base-notebook:master ./run_tests.sh base-notebook
+	docker build -t cnes/base-notebook:master . --no-cache --progress=plain --platform linux/amd64; \
+	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) cnes/base-notebook:master ./run_tests.sh base-notebook
 
 .PHONY: pangeo-notebook
 pangeo-notebook : base-image
 	cd pangeo-notebook ; \
-	conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -p linux-64 -p linux-aarch64 -p osx-64 -p osx-arm64; \
-	conda-lock render -k explicit -p linux-64; \
-	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
-	docker build -t pangeo/pangeo-notebook:master . --progress=plain --platform linux/amd64,linux/arm64 ; \
-	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/pangeo-notebook:master ./run_tests.sh pangeo-notebook
-
-.PHONY: ml-notebook
-ml-notebook : base-image
-	cd ml-notebook ; \
-	conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
+	cp -r ../base-notebook/resources . ; \
+	conda-lock lock -f environment.yml -f ../base-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
 	conda-lock render -k explicit -p linux-64; \
 	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
-	docker build -t pangeo/ml-notebook:master . ; \
-	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/ml-notebook:master ./run_tests.sh ml-notebook
+	docker build -t cnes/pangeo-notebook:master . --progress=plain --platform linux/amd64; \
+	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) cnes/pangeo-notebook:master ./run_tests.sh pangeo-notebook
 
 .PHONY: pytorch-notebook
 pytorch-notebook : base-image
 	cd pytorch-notebook ; \
 	conda-lock lock -f environment.yml -f ../pangeo-notebook/environment.yml -f ../base-notebook/environment.yml -p linux-64; \
 	conda-lock render -k explicit -p linux-64; \
 	../generate-packages-list.py conda-linux-64.lock > packages.txt; \
-	docker build -t pangeo/pytorch-notebook:master . ; \
-	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) pangeo/pytorch-notebook:master ./run_tests.sh pytorch-notebook
+	docker build -t cnes/pytorch-notebook:master . ; \
+	docker run -w $(TESTDIR) -v $(PWD):$(TESTDIR) cnes/pytorch-notebook:master ./run_tests.sh pytorch-notebook

base-image/Dockerfile

@@ -140,7 +140,7 @@ ONBUILD RUN echo "Checking for 'jupyter_notebook_config.py'..." \
         && cp jupyter_notebook_config.py /etc/jupyter \
         ; fi
 
-ONBUILD USER ${NB_USER}
+# ONBUILD USER ${NB_USER}
 
 # We want to keep our images as reproducible as possible. If a lock
 # file with exact versions of all required packages is present, we use
@@ -184,6 +184,18 @@ ONBUILD RUN echo "Checking for pip 'requirements.txt'..." \
         ${NB_PYTHON_PREFIX}/bin/pip install --no-cache -r requirements.txt \
         ; fi
 
+ONBUILD COPY resources/layer-cleanup.sh /usr/local/bin
+ONBUILD RUN chmod +x /usr/local/bin/layer-cleanup.sh
+
+ONBUILD RUN for script in $(find "resources" -type f -name "install.sh"); do \
+        chmod +x "$script"; \
+        echo "Executing $script"; \
+        bash "$script"; \
+        layer-cleanup.sh; \
+    done
+
+ONBUILD USER ${NB_USER}
+
 # If a postBuild file exists, run it!
 # After it's done, we try to remove any possible cruft commands there
 # leave behind under $HOME - particularly stuff that jupyterlab extensions
@@ -210,4 +222,4 @@ ONBUILD RUN echo "Checking for 'start'..." \
         chmod +x start \
         && cp start /srv/start \
         ; fi
-# ----------------------
+# ----------------------

base-notebook/Dockerfile

@@ -3,4 +3,4 @@
 # files (such as conda-linux-64.lock, start) in this repo.
 # Refer to the base-image/Dockerfile for documentation.
 ARG PANGEO_BASE_IMAGE_TAG=master
-FROM pangeo/base-image:${PANGEO_BASE_IMAGE_TAG}
+FROM cnes/base-image:${PANGEO_BASE_IMAGE_TAG}

base-notebook/apt.txt

@@ -0,0 +1,19 @@
+build-essential
+curl
+dfc
+file
+git
+htop
+jq
+less
+ncdu
+ranger
+rclone
+rsync
+silversearcher-ag
+software-properties-common
+tig
+tmux
+tree
+unzip
+vim