Merge branch 'develop' into bugfix/2124-fetch-template-errors

.github/actions/install-linux/action.yml

@@ -0,0 +1,78 @@
+name: Install dependencies on Linux
+description: Install dependencies on Linux
+inputs:
+  python-version:
+    description: "Python version to install"
+    required: false
+    default: "3.9"
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/[email protected]
+      with:
+        fetch-depth: 0
+    - name: Checkout repository
+      shell: bash
+      run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+    - name: Install system packages
+      shell: bash
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y libyaml-0-2 libyaml-dev unzip fuse3
+        sudo -v ; curl https://rclone.org/install.sh | sudo bash
+    - uses: actions/cache@master
+      id: dependency-cache
+      with:
+        path: |
+          ${{ env.pythonLocation }}
+          renku/templates
+        key: ${{ env.DEPENDENCY_CACHE_PREFIX }}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('Makefile') }}
+      env:
+        DEPENDENCY_CACHE_PREFIX: "v1"
+    - name: Install dependencies
+      if: steps.dependency-cache.outputs.cache-hit != 'true' || 'refs/heads/master' == github.ref || 'refs/heads/develop' == github.ref || startsWith(github.ref, 'refs/tags/')
+      env:
+        POETRY_VIRTUALENVS_CREATE: "false"
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install coveralls poetry wheel twine
+        poetry self add "poetry-dynamic-versioning[plugin]"
+        make download-templates
+        poetry install -E service --with dev,tests,docs
+    - name: Install renku into cache
+      if: steps.dependency-cache.outputs.cache-hit == 'true'
+      env:
+        POETRY_VIRTUALENVS_CREATE: "false"
+      shell: bash
+      run: |
+        poetry install -E service --with dev,tests,docs
+    - name: Set Git config
+      shell: bash
+      run: |
+        git config --global --add user.name "Renku Bot"
+        git config --global --add user.email "[email protected]"
+    - id: year-week
+      shell: bash
+      run: echo "date=$(date '+%Y-%V')" >> $GITHUB_OUTPUT
+    - name: Network cache
+      uses: actions/cache@master
+      with:
+        path: cassettes
+        key: ${{ env.NETWORK_CACHE_PREFIX }}-${{ steps.year-week.outputs.date }}-${{ hashFiles('poetry.lock') }}-${{ github.job }}
+      env:
+        NETWORK_CACHE_PREFIX: "v1"
+    - name: Set coveralls path mapping
+      shell: bash
+      run: |
+        echo "
+
+        [tool.coverage.paths]
+        source = [
+            \"renku/\",
+            \"$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')/renku/\"
+        ]" >> pyproject.toml

.github/actions/install-macos/action.yml

@@ -0,0 +1,49 @@
+name: Install dependencies on macOS
+description: Install dependencies on macOS
+inputs:
+  python-version:
+    description: "Python version to install"
+    required: false
+    default: "3.9"
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/[email protected]
+      with:
+        fetch-depth: 0
+    - name: Checkout repository
+      shell: bash
+      run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+    - name: Install dependencies
+      env:
+        POETRY_VIRTUALENVS_CREATE: "false"
+      shell: bash
+      run: |
+        brew update || true
+        brew unlink git-lfs || true
+        curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
+        brew install shellcheck node || brew link --overwrite node
+        python -m pip install --upgrade pip
+        python -m pip install wheel poetry
+        poetry self add "poetry-dynamic-versioning[plugin]"
+        make download-templates
+        poetry install -E service --with dev,tests,docs
+    - name: Set Git config
+      shell: bash
+      run: |
+        git config --global --add user.name "Renku Bot"
+        git config --global --add user.email "[email protected]"
+    - id: year-week
+      shell: bash
+      run: echo "date=$(date '+%Y-%V')" >> $GITHUB_OUTPUT
+    - name: Network cache
+      uses: actions/cache@master
+      with:
+        path: cassettes
+        key: ${{ env.NETWORK_CACHE_PREFIX }}-${{ steps.year-week.outputs.date }}-${{ hashFiles('poetry.lock') }}-${{ github.job }}
+      env:
+        NETWORK_CACHE_PREFIX: "v1"

.github/dependabot.yml

@@ -8,7 +8,6 @@ updates:
       prefix: "build(deps): "
       include: "scope"
     target-branch: "develop"
-    open-pull-requests-limit: 0
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/workflows/acceptance-tests.yml

@@ -76,7 +76,7 @@ jobs:
           body-includes: "You can access the deployment of this PR at"
       - name: Create comment pre deploy
         if: steps.findcomment.outputs.comment-id == 0
-        uses: peter-evans/create-or-update-comment@v2
+        uses: peter-evans/create-or-update-comment@v3
         with:
           token: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
           issue-number: ${{ github.event.pull_request.number }}

.github/workflows/cheatsheet.yml

@@ -6,50 +6,22 @@ on:
       - "**"
     tags:
       - "v*.*.*"
-env:
-  DEPENDENCY_CACHE_PREFIX: "v2"
-  NETWORK_CACHE_PREFIX: "v1"
+
 jobs:
   cheatsheet:
     runs-on: ubuntu-latest
     if: "'refs/heads/master' != github.ref && 'refs/heads/develop' != github.ref && !startsWith(github.ref, 'refs/tags/')"
     steps:
-      - uses: actions/[email protected]
-      - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v1
+      - uses: actions/[email protected]
         with:
-          python-version: 3.9
+          fetch-depth: 0
+      - name: Install dependencies
+        uses: ./.github/actions/install-linux
       - name: Install xetex
         run: |
           sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu jammy main restricted multiverse universe"
           sudo apt-get update -y
           sudo apt-get install -y -t jammy texlive-xetex
-
-      - uses: actions/cache@master
-        id: cache
-        with:
-          path: |
-            ${{ env.pythonLocation }}
-            renku/templates
-          key: ${{env.CACHE_PREFIX}}-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-${{hashFiles('poetry.lock')}}-${{hashFiles('build.py')}}
-      - name: Install dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        env:
-          POETRY_VIRTUALENVS_CREATE: false
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install coveralls poetry wheel twine
-          poetry self add "poetry-dynamic-versioning[plugin]"
-          poetry install -E service --with dev,tests,docs
-      - name: Install renku into cache
-        if: steps.cache.outputs.cache-hit == 'true'
-        run: |
-          python -m pip install --no-deps .
-      - name: Set Git config
-        run: |
-          git config --global --add user.name "Renku Bot"
-          git config --global --add user.email "[email protected]"
       - name: Build cheatsheet
         run: |
           sphinx-build -b latex -c docs/cheatsheet/ docs docs/latex/ -W

.github/workflows/combine-dependabot-updates.yml

@@ -1,29 +1,104 @@
 name: "Combine Dependabot PRs"
 on:
   workflow_dispatch:
+    inputs:
+      target_branch:
+        description: Target branch to create a release/PR to
+        type: string
+        required: true
+        default: develop
+      branch_name:
+        description: Name of the branch to combine PRs into
+        type: string
+        required: true
+        default: combined-prs-branch
 
 jobs:
   combine-prs:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v3.5.2
         with:
           fetch-depth: 0
           token: "${{ secrets.RENKUBOT_GITHUB_TOKEN }}"
+          ref: develop
       - name: Setup local branch and merge tool
         run: |
-          git branch --track dependabot-updates origin/dependabot-updates
+          git checkout -b "${{ inputs.branch_name }}" || ( git checkout "${{ inputs.branch_name }}" && git branch -u "origin/${{ inputs.branch_name }}" && git pull && git merge develop)
           echo "poetry.lock merge=ours" >> .git/info/attributes
           git config merge.ours.driver true
-      - uses: mark-hingston/combine-dependabot-prs@main
+          git merge --no-edit "origin/${{ inputs.target_branch }}"
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          branchPrefix: "dependabot"
-          mustBeGreen: false
-          combineBranchName: "combined-prs"
-          includeLabel: ""
-          ignoreLabel: "nocombine"
-          baseBranch: "develop"
-          openPR: true
-          allowSkipped: false
-          closeOnceCombined: true
-          githubToken: "${{ secrets.RENKUBOT_GITHUB_TOKEN }}"
+          python-version: 3.11
+      - name: Install dependencies
+        env:
+          POETRY_VIRTUALENVS_CREATE: "false"
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install poetry
+      - name: Set Git config
+        shell: bash
+        run: |
+          git config --global --add user.name "Renku Bot"
+          git config --global --add user.email "[email protected]"
+      - name: Merge PRs
+        id: merge_prs
+        env:
+          POETRY_VIRTUALENVS_CREATE: "false"
+        run: |
+          repo="https://api.github.com/repos/${{ github.repository }}"
+          dependabot_pulls=($(curl -s "$repo/pulls?state=open" | jq -r ".[] | select((.head.ref | test(\"dependabot/\"))) | \"\(.head.ref),\(.number)\""))
+
+          pr_body="# Combined PRs\n✅ The following pull requests have been successfully combined on this PR:\n"
+          failed=()
+
+          for branch in "${dependabot_pulls[@]}"; do
+            branch_arr=(${branch//,/ })
+            echo "Merging branch ${branch_arr[0]}"
+
+            if git merge --no-edit "origin/${branch_arr[0]}"; then
+              pr_body="$pr_body\n- #${branch_arr[1]}"
+            else
+              git merge --abort
+              git clean -fdx
+              git reset --hard
+              failed+=(${branch_arr[1]})
+            fi
+          done
+
+          if (( ${#failed[@]} )); then
+            pr_body="$pr_body\n\nFailed to combine:\n"
+
+            for fail in "${failed[@]}"; do
+              pr_body="$pr_body\n- #$fail"
+            done
+          fi
+          poetry lock
+          git add -A
+          git commit -m "update lock file" --no-verify
+          git status
+          git push origin "${{ inputs.branch_name }}"
+          echo "pr_body=$pr_body" >> $GITHUB_OUTPUT
+      - name: Create Pull Request
+        uses: actions/github-script@v6
+        with:
+          token: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
+          script: |
+            const { repo, owner } = context.repo;
+            const result = await github.rest.pulls.create({
+              title: 'chore: combined dependency update',
+              owner,
+              repo,
+              head: '${{ inputs.branch_name }}',
+              base: '${{ inputs.target_branch }}',
+              body: '${{ steps.merge_prs.outputs.pr_body }}'
+            });
+            github.rest.issues.addLabels({
+              owner,
+              repo,
+              issue_number: result.data.number,
+              labels: ['dependencies', 'automated pr']
+            });

.github/workflows/publish_shacl.yml

@@ -11,26 +11,21 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/[email protected]
-      with:
-        fetch-depth: 0
-    - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.10
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install .[all]
-    - name: Publish shape
-      run: |
-        git clone https://${{secrets.RENKUBOT_GITHUB_TOKEN}}@github.com/SwissDataScienceCenter/renku-ontology.git shacl_publish
-        cd shacl_publish
-        mkdir $(renku --version)
-        cp ../renku/data/shacl_shape.json ./$(renku --version)/
-        git config --local user.email "[email protected]"
-        git config --local user.name "RenkuBot"
-        git add $(renku --version)/shacl_shape.json
-        git commit -m "Publish SHACL shape for $(renku --version)"
-        git push
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+      - name: Install dependencies
+        uses: ./.github/actions/install-linux
+        with:
+          python-version: "3.10"
+      - name: Publish shape
+        run: |
+          git clone https://${{secrets.RENKUBOT_GITHUB_TOKEN}}@github.com/SwissDataScienceCenter/renku-ontology.git shacl_publish
+          cd shacl_publish
+          mkdir $(renku --version)
+          cp ../renku/data/shacl_shape.json ./$(renku --version)/
+          git config --local user.email "[email protected]"
+          git config --local user.name "RenkuBot"
+          git add $(renku --version)/shacl_shape.json
+          git commit -m "Publish SHACL shape for $(renku --version)"
+          git push

.github/workflows/pypi-release.yml

@@ -11,7 +11,7 @@ jobs:
   finalize-release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v3.5.2
         with:
           fetch-depth: 0
           token: "${{ secrets.RENKUBOT_GITHUB_TOKEN }}"

.github/workflows/release.yml

@@ -21,7 +21,7 @@ jobs:
   create-release-pr:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v3.5.2
         with:
           fetch-depth: 0
           token: "${{ secrets.RENKUBOT_GITHUB_TOKEN }}"

.github/workflows/semantic_pr.yml

@@ -10,6 +10,6 @@ jobs:
   main:
     runs-on: ubuntu-latest
     steps:
-      - uses: amannn/action-semantic-pull-request@v5.1.0
+      - uses: amannn/action-semantic-pull-request@v5.2.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}