From fff20ba40bed3a3dcc5a0c1620f0ee9cc6570161 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Fri, 10 Feb 2023 23:42:47 +0100 Subject: [PATCH 01/13] chore: update release action to support arbitrary refs (#3268) --- .github/workflows/release.yml | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 65bd356d5d..3794128937 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,16 @@ on: description: New release version type: string required: true + ref: + description: Branch/ref to create a release from + type: string + required: true + default: develop + target_branch: + description: Target branch to create a release to + type: string + required: true + default: master jobs: create-release-pr: @@ -15,12 +25,12 @@ jobs: with: fetch-depth: 0 token: "${{ secrets.RENKUBOT_GITHUB_TOKEN }}" - ref: 'develop' + ref: '${{ github.event.inputs.ref }}' - uses: actions/setup-node@v3 - name: Create Branch run: git checkout -b "release/v${{ github.event.inputs.version }}" - - name: Merge Master - run: git merge origin/master + - name: Merge Target + run: git merge "origin/${{ github.event.inputs.target_branch }}" - name: Install dependencies run: | sudo add-apt-repository -y ppa:rmescandon/yq @@ -31,8 +41,8 @@ jobs: id: changelog run: | echo '{"version": "${{ github.event.inputs.version }}"}' > context.json - conventional-changelog -r 1 -p angular -c context.json | pandoc --from markdown --to rst | sed -e '/=======\n/r /dev/stdin' -e 's/=======\n/=======\n\n/' -i CHANGES.rst - conventional-changelog -r 1 -p angular -c context.json > release_body.md + conventional-changelog -r 1 -p angular -c context.json | sed -e '1,${s/^# /## /}' | pandoc --from markdown --to rst | sed -e '/=======/r /dev/stdin' -e '0,/=======/ s/=======/=======\n/' -i CHANGES.rst + conventional-changelog -r 1 -p angular -c context.json | sed -e '1,${s/^# /## /}' > release_body.md rm context.json - name: Update Chart version run: | @@ -55,7 +65,7 @@ jobs: bodyFile: "release_body.md" tag: "v${{ github.event.inputs.version }}" name: "v${{ github.event.inputs.version }}" - commit: "master" + commit: "${{ github.event.inputs.target_branch }}" - name: Create Pull Request uses: actions/github-script@v6 with: @@ -67,7 +77,7 @@ jobs: owner, repo, head: 'release/v${{ github.event.inputs.version }}', - base: 'master', + base: '${{ github.event.inputs.target_branch }}', body: [ 'This PR is auto-generated by', '[actions/github-script](https://github.com/actions/github-script).' From 5cac2a351424c4f3a2f8c4f18603699045dc4c6a Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Mon, 13 Feb 2023 16:46:23 +0100 Subject: [PATCH 02/13] chore: update issue template for issue triage (#3309) --- .github/ISSUE_TEMPLATE/bug_report.md | 6 ++- .github/ISSUE_TEMPLATE/config.yml | 11 +++++ .github/ISSUE_TEMPLATE/documentation.md | 13 ++++++ .github/ISSUE_TEMPLATE/feature_request.md | 6 ++- .github/ISSUE_TEMPLATE/improvement_request.md | 10 ----- .github/ISSUE_TEMPLATE/proposal.md | 45 ------------------- .github/workflows/github-project.yml | 9 +++- 7 files changed, 40 insertions(+), 60 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/documentation.md delete mode 100644 .github/ISSUE_TEMPLATE/improvement_request.md delete mode 100644 .github/ISSUE_TEMPLATE/proposal.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index b79971b097..0f738ec576 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,9 @@ --- -name: Bug report +name: "\U0001F41E Bug report" about: Create a report to help us improve - +title: '' +labels: kind/bug, status/triage +assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..66beb21d62 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +# Ref: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository#configuring-the-template-chooser +blank_issues_enabled: false +contact_links: +- name: '💬 Discussions' + url: https://renku.discourse.group/ + about: | + Ask questions about using Renku, its features and roadmap, or get support and give feedback. +- name: '💬 Gitter' + url: https://gitter.im/SwissDataScienceCenter/renku + about: | + Chat with the community and maintainers about both the usage of and development of the project. diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000000..07deca99dc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,13 @@ +--- +name: "\U0001F4DA Documentation Issue" +about: Did you find errors, difficulties or missing information in the documentation? +title: '' +labels: kind/documentation, status/triage +assignees: '' + +--- + + + +**Describe the issue with documentation you found** +A clear and concise description of what is missing or should be added/improved. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 3074ca6bb9..399953a1db 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,7 +1,9 @@ --- -name: Feature request +name: "\U0001F381 Feature request" about: Suggest an idea for this project - +title: '' +labels: kind/enhancement, status/triage +assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/improvement_request.md b/.github/ISSUE_TEMPLATE/improvement_request.md deleted file mode 100644 index 4b3c89c846..0000000000 --- a/.github/ISSUE_TEMPLATE/improvement_request.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: Improvement request -about: Suggest an improvement/optimization for this project - ---- - - - -**Is your improvement request related to a use case or a component design? Please describe.** -A clear and concise description of what the use case is. diff --git a/.github/ISSUE_TEMPLATE/proposal.md b/.github/ISSUE_TEMPLATE/proposal.md deleted file mode 100644 index 53a54f52e1..0000000000 --- a/.github/ISSUE_TEMPLATE/proposal.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -name: Proposal -about: Submit a proposal for open discussion - ---- - - - -[This is a template for Renku's change proposal process.] - -# Proposal: [Title] - -Author(s): [Author Name, Co-Author Name] - -Last updated: [Date] - -Discussion at https://github.com/SwissDataScienceCenter/renku-python/issue/NNNNN. - -## Abstract - -[A short summary of the proposal.] - -## Background - -[An introduction of the necessary background and the problem being solved by the proposed change.] - -## Proposal - -[A precise statement of the proposed change.] - -## Rationale - -[A discussion of alternate approaches and the trade offs, advantages, and disadvantages of the specified approach.] - -## Compatibility - -## Implementation - -[A description of the steps in the implementation, who will do them, and when. -This should include a discussion of how the work fits into release cycle.] - -## Open issues (if applicable) - -[A discussion of issues relating to this proposal for which the author does not -know the solution. This section may be omitted if there are none.] diff --git a/.github/workflows/github-project.yml b/.github/workflows/github-project.yml index 09fdf7935d..8080d57026 100644 --- a/.github/workflows/github-project.yml +++ b/.github/workflows/github-project.yml @@ -10,7 +10,14 @@ jobs: add_to_project: runs-on: ubuntu-latest steps: - - uses: actions/add-to-project@main + - name: Add to renku-python + uses: actions/add-to-project@main with: project-url: https://github.com/orgs/SwissDataScienceCenter/projects/45 github-token: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} + - name: Add to triage board + uses: actions/add-to-project@main + with: + project-url: https://github.com/orgs/SwissDataScienceCenter/projects/51 + github-token: ${{ secrets.RENKUBOT_GITHUB_TOKEN }} + labeled: status/triage From 1fd8082595c0ebf320a4512652a128249cad3519 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Tue, 14 Feb 2023 14:41:12 +0100 Subject: [PATCH 03/13] docs: fix RTD config (#3316) --- .readthedocs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f904aa70fc..4f8ce961dd 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,7 +19,6 @@ build: - poetry install --with docs --with tests python: - version: 3.9 install: - method: pip path: . From 0e5906373b875f8f8c5d7ba1824f8794e2c31045 Mon Sep 17 00:00:00 2001 From: RenkuBot <53332360+RenkuBot@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:53:41 +0100 Subject: [PATCH 04/13] Update combined dependencies (#3317) --- .github/workflows/semantic_pr.yml | 2 +- poetry.lock | 234 ++++++++++++++++-------------- pyproject.toml | 2 +- 3 files changed, 126 insertions(+), 112 deletions(-) diff --git a/.github/workflows/semantic_pr.yml b/.github/workflows/semantic_pr.yml index efec5b50e8..3b57a4edf2 100644 --- a/.github/workflows/semantic_pr.yml +++ b/.github/workflows/semantic_pr.yml @@ -10,6 +10,6 @@ jobs: main: runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v5.0.2 + - uses: amannn/action-semantic-pull-request@v5.1.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/poetry.lock b/poetry.lock index 93fc2fe7b6..2410b7f024 100644 --- a/poetry.lock +++ b/poetry.lock @@ -215,14 +215,14 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "blessed" -version = "1.19.1" +version = "1.20.0" description = "Easy, practical library for making terminal apps, by providing an elegant, well-documented interface to Colors, Keyboard input, and screen Positioning capabilities." category = "main" optional = false python-versions = ">=2.7" files = [ - {file = "blessed-1.19.1-py2.py3-none-any.whl", hash = "sha256:63b8554ae2e0e7f43749b6715c734cc8f3883010a809bf16790102563e6cf25b"}, - {file = "blessed-1.19.1.tar.gz", hash = "sha256:9a0d099695bf621d4680dd6c73f6ad547f6a3442fbdbe80c4b1daa1edbc492fc"}, + {file = "blessed-1.20.0-py2.py3-none-any.whl", hash = "sha256:0c542922586a265e699188e52d5f5ac5ec0dd517e5a1041d90d2bbf23f906058"}, + {file = "blessed-1.20.0.tar.gz", hash = "sha256:2cdd67f8746e048f00df47a2880f4d6acbcdb399031b604e34ba8f71d5787680"}, ] [package.dependencies] @@ -244,48 +244,42 @@ files = [ [[package]] name = "btrees" -version = "4.11.3" +version = "5.0" description = "Scalable persistent object containers" category = "main" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +python-versions = ">=3.7" files = [ - {file = "BTrees-4.11.3-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:6e66d14f6eb5cae34d71be7c948b848bbb22761cafe7e1966b018c4a44605492"}, - {file = "BTrees-4.11.3-cp27-cp27m-win_amd64.whl", hash = "sha256:7475bd63d6a588f89caa5a21fe1b8a408522f83213b840fb5308ea366edfa2ef"}, - {file = "BTrees-4.11.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a18c3cbefa304cff31a5ba2d286b1b6e00676bfe25e00a5f4020cefaee728339"}, - {file = "BTrees-4.11.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:16b1b814e9046257465d953a12696b0610aa5c3571fea604984f87307dfcb9f2"}, - {file = "BTrees-4.11.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19a189532671a049c5098ff84d0f4fa3c43e315bf680b9fbdd149c26c6d6d133"}, - {file = "BTrees-4.11.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:911947a8849b6918a8b8d961684bd4f48075831afa5c75c6938ea6e729d87ae1"}, - {file = "BTrees-4.11.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7807f918d496af5d5b69d527bdaede5b97acbd8f0bead7eb6b14cd042a21df42"}, - {file = "BTrees-4.11.3-cp310-cp310-win_amd64.whl", hash = "sha256:817c68833bb924fc017ee5db40ecb6e5e25c1089568453a78680a37fb56929ed"}, - {file = "BTrees-4.11.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fdeeba5c3ec4399e9b7ca7664e3719bf958b599bdfca2b660d3dc4f49519995b"}, - {file = "BTrees-4.11.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:df487853e5a79dfc61e4a71c0b2e281525b265d5de2af3794960621de40dd2c6"}, - {file = "BTrees-4.11.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cb4caf3c78a285c1a68effbadca3f3f1f0270a7e9064d18ef4fbab25bf22188"}, - {file = "BTrees-4.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:aecc3daf36a1f08f2f40aa36770121bf2864e0a3c928eff6d7a7fc815a2ca095"}, - {file = "BTrees-4.11.3-cp35-cp35m-win_amd64.whl", hash = "sha256:026c3827bdc034f3a283c573ce983a0b860cef1e8f920b82a25bd871c22bd17c"}, - {file = "BTrees-4.11.3-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:a4746aee3c5eadc114664fbc951fa534b2387853d36113972f65235b9cb20438"}, - {file = "BTrees-4.11.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1024837164fffaa6c426eca86d0a56dadc6a0fa23e97fda77046e7a2d462366"}, - {file = "BTrees-4.11.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:3217969fe803ea55dc606118eda7124162ca2c3e3a501dc89302201a5c30eaba"}, - {file = "BTrees-4.11.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:54a4223a18a597f5e86ce8bd3787e4aae124bf44e8ed80bf82043fe381090542"}, - {file = "BTrees-4.11.3-cp36-cp36m-win_amd64.whl", hash = "sha256:c41d8b7998095c42e8f896ee659ba35adda403203e447269800198529800979e"}, - {file = "BTrees-4.11.3-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:ef55f2f652c849c18d3ad09bd5f0c0998868124dbecb07236fa4021d0246ae67"}, - {file = "BTrees-4.11.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36c2602a45f9e1fc4147be85c5efc67923fbe01d81174fd0de305e75d05116bf"}, - {file = "BTrees-4.11.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9fa6dcdc874526662eaa71ff20fb43fadb3c7636cf951ba6bb58b30a82020637"}, - {file = "BTrees-4.11.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:85cdcbd1533fab59c6e065ad86b6913e539b8b0f2a2bd084faa5a2f09b8fcb43"}, - {file = "BTrees-4.11.3-cp37-cp37m-win_amd64.whl", hash = "sha256:fcd172812ba76d360cf3c08f78029837ff71a4b6f0d05e6cef4cd2bc9feeb9a7"}, - {file = "BTrees-4.11.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:163c44787fd0959f19e18968abce6a2e6d6e1c24cf4d2e198475d9b7917a59b1"}, - {file = "BTrees-4.11.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cbea0734e22500bcd70856ffa3e22ff06c968199ac5c0e6701d1b21db821e0c"}, - {file = "BTrees-4.11.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57d090057954f5b32ea69f09aa3e85e75f89d28a2090320519e084dc0adedc61"}, - {file = "BTrees-4.11.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9c5be0a2e9e1e60d357a00371cf9c064e0a5092ed2cfe7f8bf245b506bc5c981"}, - {file = "BTrees-4.11.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4ae5fd4f45fb6510b8d76c285c7cc82325139b04c988d1bcee6e9754846e2c"}, - {file = "BTrees-4.11.3-cp38-cp38-win_amd64.whl", hash = "sha256:3f48008b385aaabdab6dd589a725be36a4455ebdc1c2d2bf2881bc17f5fa696f"}, - {file = "BTrees-4.11.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c193cf41355d7780c6bc2e752608950aa01a17b70d4ad836241d38545e97a528"}, - {file = "BTrees-4.11.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bd466a994604865286c3933e1c398d410fd39580d8a9c7afb01b45c4cf078c89"}, - {file = "BTrees-4.11.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5508f5cad1f45a5d63112f883dd917a42fabf48a5f69615a0d8de6181e8950cf"}, - {file = "BTrees-4.11.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8356448ba81f05cfc37eb7d47f2c0202b8dabf293dc0bb6fc6c8c4dda3c61322"}, - {file = "BTrees-4.11.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aa05448e6a43d7291558d169a0e483f70feacbd2c3dda70f00d31b71a88f2cb5"}, - {file = "BTrees-4.11.3-cp39-cp39-win_amd64.whl", hash = "sha256:4c0d18bf999b12dd380f9909fe93d093d3b273ad9ff0458781d3bdf75d0b3214"}, - {file = "BTrees-4.11.3.tar.gz", hash = "sha256:908500b020ff989b00946f8a6f6573f38a9b1808d077e52e3dcf048fb170c13a"}, + {file = "BTrees-5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:031b652bb9ef763b26938d5d9529b3354bb46c2a20705860fe2a601d2243b533"}, + {file = "BTrees-5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4adc985ce7c98846917798d41b8fd658ede48d6fbe6e28c07a1f88c0f3a40435"}, + {file = "BTrees-5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9df85c90227a77094e1b87b4f7e59898a5e83372f697830f60e9307a51b45aeb"}, + {file = "BTrees-5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f74322b18ecfe5cf62170b0c3d47f6ecdfbef8ac3c7b9363a4f1958e4c1b36"}, + {file = "BTrees-5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:838a8a0ea1929cc4b574ac2d836841608725d2640fd996e634a28a35b22b26d8"}, + {file = "BTrees-5.0-cp310-cp310-win_amd64.whl", hash = "sha256:360acdbfb994d3aa33e39a2d1fc3d9a4cc4a21dce94cb7b0e67142600c500599"}, + {file = "BTrees-5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0a36e4cd00e33a28e6ce46c7da20a4bc6819674808516cfceb8867e47892b6db"}, + {file = "BTrees-5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4cd1fb9746045ed19fecab82f96cd7f8e582686dd2b7b25fba324f227bed1a8"}, + {file = "BTrees-5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa99719c29c3e1f03c0bf64bdcfd0148e052804408bc54f655bbb849b1e90ae6"}, + {file = "BTrees-5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:427abe3aeec418a48bf1c32fcc5a4b341abb07d3fad5373587e6dae0e15646a9"}, + {file = "BTrees-5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5410acbf1020382ab5054dcce5cf3a7dd85e52f0aa7b4e97a0fc6112fcefdebd"}, + {file = "BTrees-5.0-cp311-cp311-win_amd64.whl", hash = "sha256:2e2095eaab0262e8d288e42c23a074c0e40753c4337d18a85bb69351a3fdeaf8"}, + {file = "BTrees-5.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:2931451e57f115e7a06b3d2a6696e188bf70927524d99c07ee2a5cf1b2c0908f"}, + {file = "BTrees-5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a01f443c80afd222ef391c1cef672f6954d462ecc96005f54994b33dcaa6411"}, + {file = "BTrees-5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73bd5941e7800a39d79373f565ca28259f29605148048c41d679dd5ba0df400c"}, + {file = "BTrees-5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7fb9ab3cddad299221d8884f837a2f0738b9cc8c76651300bd7974a6dfae86ec"}, + {file = "BTrees-5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:defbe27d72a300b5d5ed1523924450ff265f5e9a20ef23f30c8ef77567f32668"}, + {file = "BTrees-5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2216fb6fd74764ac848b056bcb4ad47c60e33b2c091b4a3c5efc6aca940ea961"}, + {file = "BTrees-5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:88d4c57c2bfbea2b426c226c77533647d8d52f266c6757f0636673cbbac15c48"}, + {file = "BTrees-5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:581c97decf0636e41345faa4becb8373fdc0fdb6373e7042b422b7ae3a4bcdab"}, + {file = "BTrees-5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c6c23c93e44eb07ed82276d4a9e5a45fe33617f9ebbc5e42b17d09e16d4add2"}, + {file = "BTrees-5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e54888ffcc5408d533b069e1735eebab1843414629d9f266199ea77ab4ae9a3"}, + {file = "BTrees-5.0-cp38-cp38-win_amd64.whl", hash = "sha256:c92fdb98cb1d5de493000a934cb21a64ea218beaaf6fb0776d53818682a6523a"}, + {file = "BTrees-5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:95f084e5f6e0f966604376e500c31886f604166f2bc5679f33b5f246b7b41a33"}, + {file = "BTrees-5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:42c89ce6392a87a673ae4d7cacf6c4ce7e47448f7424bc3988d5bddc1a853401"}, + {file = "BTrees-5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27db53c743a269fa71e9bf081b2f027ec9d93695794685e82b8149b5398049ff"}, + {file = "BTrees-5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c89552dfe833749476c5fe992e7c542dca643257f6d9d91ddd259cc53faac1f"}, + {file = "BTrees-5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb455d247f8247f90ef057e61eea6ba072037efaf3947978897fde63f90c29ff"}, + {file = "BTrees-5.0-cp39-cp39-win_amd64.whl", hash = "sha256:95993d456b2a8f5dae8117bbf6a0ba6efa07d72055b24088eb3e40ef53f46473"}, + {file = "BTrees-5.0.tar.gz", hash = "sha256:fddf8a4dcc0253e39027bab380a2e20d47ae370a32485416f2a66ad9ae1a43fa"}, ] [package.dependencies] @@ -668,50 +662,49 @@ python-dateutil = "*" [[package]] name = "cryptography" -version = "38.0.4" +version = "39.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, - {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b"}, - {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"}, - {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00"}, - {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0"}, - {file = "cryptography-38.0.4-cp36-abi3-win32.whl", hash = "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744"}, - {file = "cryptography-38.0.4-cp36-abi3-win_amd64.whl", hash = "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7"}, - {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876"}, - {file = "cryptography-38.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee"}, - {file = "cryptography-38.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9"}, - {file = "cryptography-38.0.4.tar.gz", hash = "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290"}, + {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965"}, + {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f"}, + {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106"}, + {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c"}, + {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4"}, + {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, + {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"}, + {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6"}, + {file = "cryptography-39.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a"}, + {file = "cryptography-39.0.1.tar.gz", hash = "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695"}, ] [package.dependencies] cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +pep8test = ["black", "check-manifest", "mypy", "ruff", "types-pytz", "types-requests"] sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist", "pytz"] +test-randomorder = ["pytest-randomly"] +tox = ["tox"] [[package]] name = "cwl-upgrader" @@ -916,14 +909,14 @@ packaging = ">=20.9" [[package]] name = "enlighten" -version = "1.11.1" +version = "1.11.2" description = "Enlighten Progress Bar" category = "main" optional = false python-versions = "*" files = [ - {file = "enlighten-1.11.1-py2.py3-none-any.whl", hash = "sha256:e825eb534ca80778bb7d46e5581527b2a6fae559b6cf09e290a7952c6e11961e"}, - {file = "enlighten-1.11.1.tar.gz", hash = "sha256:57abd98a3d3f83484ef9f91f9255f4d23c8b3097ecdb647c7b9b0049d600b7f8"}, + {file = "enlighten-1.11.2-py2.py3-none-any.whl", hash = "sha256:98c9eb20e022b6a57f1c8d4f17e16760780b6881e6d658c40f52d21255ea45f3"}, + {file = "enlighten-1.11.2.tar.gz", hash = "sha256:9284861dee5a272e0e1a3758cd3f3b7180b1bd1754875da76876f2a7f46ccb61"}, ] [package.dependencies] @@ -1233,14 +1226,14 @@ tests = ["freezegun", "pytest", "pytest-cov"] [[package]] name = "identify" -version = "2.5.15" +version = "2.5.18" description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "identify-2.5.15-py2.py3-none-any.whl", hash = "sha256:1f4b36c5f50f3f950864b2a047308743f064eaa6f6645da5e5c780d1c7125487"}, - {file = "identify-2.5.15.tar.gz", hash = "sha256:c22aa206f47cc40486ecf585d27ad5f40adbfc494a3fa41dc3ed0499a23b123f"}, + {file = "identify-2.5.18-py2.py3-none-any.whl", hash = "sha256:93aac7ecf2f6abf879b8f29a8002d3c6de7086b8c28d88e1ad15045a15ab63f9"}, + {file = "identify-2.5.18.tar.gz", hash = "sha256:89e144fa560cc4cffb6ef2ab5e9fb18ed9f9b3cb054384bab4b95c12f6c309fe"}, ] [package.extras] @@ -1607,6 +1600,7 @@ files = [ {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca989b91cf3a3ba28930a9fc1e9aeafc2a395448641df1f387a2d394638943b0"}, {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:822068f85e12a6e292803e112ab876bc03ed1f03dddb80154c395f891ca6b31e"}, {file = "lxml-4.9.2-cp35-cp35m-win32.whl", hash = "sha256:be7292c55101e22f2a3d4d8913944cbea71eea90792bf914add27454a13905df"}, + {file = "lxml-4.9.2-cp35-cp35m-win_amd64.whl", hash = "sha256:998c7c41910666d2976928c38ea96a70d1aa43be6fe502f21a651e17483a43c5"}, {file = "lxml-4.9.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:b26a29f0b7fc6f0897f043ca366142d2b609dc60756ee6e4e90b5f762c6adc53"}, {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:ab323679b8b3030000f2be63e22cdeea5b47ee0abd2d6a1dc0c8103ddaa56cd7"}, {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:689bb688a1db722485e4610a503e3e9210dcc20c520b45ac8f7533c837be76fe"}, @@ -1616,6 +1610,7 @@ files = [ {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:58bfa3aa19ca4c0f28c5dde0ff56c520fbac6f0daf4fac66ed4c8d2fb7f22e74"}, {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc718cd47b765e790eecb74d044cc8d37d58562f6c314ee9484df26276d36a38"}, {file = "lxml-4.9.2-cp36-cp36m-win32.whl", hash = "sha256:d5bf6545cd27aaa8a13033ce56354ed9e25ab0e4ac3b5392b763d8d04b08e0c5"}, + {file = "lxml-4.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:3ab9fa9d6dc2a7f29d7affdf3edebf6ece6fb28a6d80b14c3b2fb9d39b9322c3"}, {file = "lxml-4.9.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:05ca3f6abf5cf78fe053da9b1166e062ade3fa5d4f92b4ed688127ea7d7b1d03"}, {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:a5da296eb617d18e497bcf0a5c528f5d3b18dadb3619fbdadf4ed2356ef8d941"}, {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:04876580c050a8c5341d706dd464ff04fd597095cc8c023252566a8826505726"}, @@ -2002,14 +1997,14 @@ reports = ["lxml"] [[package]] name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." category = "main" optional = false -python-versions = "*" +python-versions = ">=3.5" files = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] [[package]] @@ -2339,6 +2334,25 @@ typing-extensions = {version = ">=4.4", markers = "python_version < \"3.8\""} docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +[[package]] +name = "platformdirs" +version = "3.0.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.0.0-py3-none-any.whl", hash = "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567"}, + {file = "platformdirs-3.0.0.tar.gz", hash = "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.4", markers = "python_version < \"3.8\""} + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + [[package]] name = "pluggy" version = "1.0.0" @@ -2976,14 +2990,14 @@ pytest = ">=3.5" [[package]] name = "pytest-forked" -version = "1.4.0" +version = "1.6.0" description = "run tests in isolated forked subprocesses" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "pytest-forked-1.4.0.tar.gz", hash = "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e"}, - {file = "pytest_forked-1.4.0-py3-none-any.whl", hash = "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8"}, + {file = "pytest-forked-1.6.0.tar.gz", hash = "sha256:4dafd46a9a600f65d822b8f605133ecf5b3e1941ebb3588e943b4e3eb71a5a3f"}, + {file = "pytest_forked-1.6.0-py3-none-any.whl", hash = "sha256:810958f66a91afb1a1e2ae83089d8dc1cd2437ac96b12963042fbb9fb4d16af0"}, ] [package.dependencies] @@ -3634,14 +3648,14 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "66.1.1" +version = "67.3.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-66.1.1-py3-none-any.whl", hash = "sha256:6f590d76b713d5de4e49fe4fbca24474469f53c83632d5d0fd056f7ff7e8112b"}, - {file = "setuptools-66.1.1.tar.gz", hash = "sha256:ac4008d396bc9cd983ea483cb7139c0240a07bbc74ffb6232fceffedc6cf03a8"}, + {file = "setuptools-67.3.1-py3-none-any.whl", hash = "sha256:23c86b4e44432bfd8899384afc08872ec166a24f48a3f99f293b0a557e6a6b5d"}, + {file = "setuptools-67.3.1.tar.gz", hash = "sha256:daec07fd848d80676694d6bf69c009d28910aeece68a38dbe88b7e1bb6dba12e"}, ] [package.extras] @@ -4172,14 +4186,14 @@ files = [ [[package]] name = "types-pyyaml" -version = "6.0.12.3" +version = "6.0.12.6" description = "Typing stubs for PyYAML" category = "dev" optional = false python-versions = "*" files = [ - {file = "types-PyYAML-6.0.12.3.tar.gz", hash = "sha256:17ce17b3ead8f06e416a3b1d5b8ddc6cb82a422bb200254dd8b469434b045ffc"}, - {file = "types_PyYAML-6.0.12.3-py3-none-any.whl", hash = "sha256:879700e9f215afb20ab5f849590418ab500989f83a57e635689e1d50ccc63f0c"}, + {file = "types-PyYAML-6.0.12.6.tar.gz", hash = "sha256:24e76b938d58e68645271eeb149af6022d1da99788e481f959bd284b164f39a1"}, + {file = "types_PyYAML-6.0.12.6-py3-none-any.whl", hash = "sha256:77b74d0874482f2b42dd566b7277b0a220068595e0fb42689d0c0560f3d1ae9e"}, ] [[package]] @@ -4220,14 +4234,14 @@ files = [ [[package]] name = "types-toml" -version = "0.10.8.1" +version = "0.10.8.3" description = "Typing stubs for toml" category = "dev" optional = false python-versions = "*" files = [ - {file = "types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, - {file = "types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, + {file = "types-toml-0.10.8.3.tar.gz", hash = "sha256:f37244eff4cd7eace9cb70d0bac54d3eba77973aa4ef26c271ac3d1c6503a48e"}, + {file = "types_toml-0.10.8.3-py3-none-any.whl", hash = "sha256:a2286a053aea6ab6ff814659272b1d4a05d86a1dd52b807a87b23511993b46c5"}, ] [[package]] @@ -4301,25 +4315,25 @@ testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", [[package]] name = "virtualenv" -version = "20.17.1" +version = "20.19.0" description = "Virtual Python Environment builder" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, - {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, + {file = "virtualenv-20.19.0-py3-none-any.whl", hash = "sha256:54eb59e7352b573aa04d53f80fc9736ed0ad5143af445a1e539aada6eb947dd1"}, + {file = "virtualenv-20.19.0.tar.gz", hash = "sha256:37a640ba82ed40b226599c522d411e4be5edb339a0c0de030c0dc7b646d61590"}, ] [package.dependencies] distlib = ">=0.3.6,<1" filelock = ">=3.4.1,<4" importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.8\""} -platformdirs = ">=2.4,<3" +platformdirs = ">=2.4,<4" [package.extras] -docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] -testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] +test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23)", "pytest (>=7.2.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)"] [[package]] name = "walrus" @@ -4349,14 +4363,14 @@ files = [ [[package]] name = "websocket-client" -version = "1.5.0" +version = "1.5.1" description = "WebSocket client for Python with low level API options" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "websocket-client-1.5.0.tar.gz", hash = "sha256:561ca949e5bbb5d33409a37235db55c279235c78ee407802f1d2314fff8a8536"}, - {file = "websocket_client-1.5.0-py3-none-any.whl", hash = "sha256:fb5d81b95d350f3a54838ebcb4c68a5353bbd1412ae8f068b1e5280faeb13074"}, + {file = "websocket-client-1.5.1.tar.gz", hash = "sha256:3f09e6d8230892547132177f575a4e3e73cfdf06526e20cc02aa1c3b47184d40"}, + {file = "websocket_client-1.5.1-py3-none-any.whl", hash = "sha256:cdf5877568b7e83aa7cf2244ab56a3213de587bbe0ce9d8b9600fc77b455d89e"}, ] [package.extras] @@ -4680,18 +4694,18 @@ uvloop = ["uvloop (>=0.5.1)"] [[package]] name = "zipp" -version = "3.11.0" +version = "3.13.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, - {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, + {file = "zipp-3.13.0-py3-none-any.whl", hash = "sha256:e8b2a36ea17df80ffe9e2c4fda3f693c3dad6df1697d3cd3af232db680950b0b"}, + {file = "zipp-3.13.0.tar.gz", hash = "sha256:23f70e964bc11a34cef175bc90ba2914e1e4545ea1e3e2f67c079671883f9cb6"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [[package]] @@ -4931,4 +4945,4 @@ service = ["apispec", "apispec-webframeworks", "circus", "flask", "gunicorn", "m [metadata] lock-version = "2.0" python-versions = "^3.7.1" -content-hash = "b1e8a803183bfcc430dc9992e57f1113594a91bb99bd639c0e196ada0fa6d310" +content-hash = "bf88419b3f3d9fa85eb5d1e37ac16c508abd5ec3a378a377e002cee17f6b3e4b" diff --git a/pyproject.toml b/pyproject.toml index 7b3813af1d..43fe28a8ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ click = ">=8.0,<8.1.4" click-option-group = "<0.6.0,>=0.5.2" click-plugins = "==1.1.1" coverage = { version = "<6.5,>=4.5.3", extras=["toml"], optional = true } -cryptography = "<39.0.0,>=38.0.0" +cryptography = ">=38.0.0,<40.0.0" cwl-utils = ">=0.12,<0.18" cwltool = "==3.1.20220628170238" deepdiff = "^5.8.0" From 5898cbb5756c86013c169e234e8002285e4e84a7 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Wed, 15 Feb 2023 12:16:48 +0100 Subject: [PATCH 05/13] fix(cli): fix renku workflow visualize crashing for large graphs due to a curses error (#3273) --- renku/ui/cli/utils/curses.py | 46 +++++++++++++++++++++++++++++------- renku/ui/cli/workflow.py | 4 ++-- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/renku/ui/cli/utils/curses.py b/renku/ui/cli/utils/curses.py index 8afed79fbd..5e5d83ad98 100644 --- a/renku/ui/cli/utils/curses.py +++ b/renku/ui/cli/utils/curses.py @@ -22,6 +22,7 @@ from renku.command.view_model.text_canvas import Point from renku.core import errors +from renku.core.util import communication from renku.domain_model.provenance.activity import Activity @@ -59,13 +60,14 @@ def __init__( self.current_layer = 0 self.layer_position = 0 self.max_layer = len(navigation_data) - 1 - self.y_pos = 0 - self.x_pos = 0 + self.y_pos: int = 0 + self.x_pos: int = 0 self.color_cache: Dict[str, int] = {} self.activity_overlay: Optional[curses._CursesWindow] = None self.help_overlay: Optional[curses._CursesWindow] = None self._select_activity() self.use_color = use_color + self.free_move: bool = False def _init_curses(self, screen): """Initialize curses screen for interactive mode.""" @@ -88,7 +90,17 @@ def _init_curses(self, screen): text_data_lines = self.text_data.splitlines() self.content_max_x = max(len(line) for line in text_data_lines) - self.content_max_y = len(self.text_data) + self.content_max_y = len(text_data_lines) + + int16_max = 32767 + if self.content_max_y > int16_max or self.content_max_x > int16_max: + communication.warn( + f"Graph is too large for interactive visualization, cropping to {int16_max} lines/columns." + ) + self.content_max_x = min(self.content_max_x, int16_max) + self.content_max_y = min(self.content_max_y, int16_max) + text_data_lines = [line[: self.content_max_x] for line in text_data_lines[self.content_max_y]] + self.content_pad = curses.newpad(self.content_max_y, self.content_max_x) for i, l in enumerate(text_data_lines): self._addstr_with_color_codes(self.content_pad, i, 0, l) @@ -281,6 +293,7 @@ def _update_help_overlay(self, screen): content = ( "Navigate using arrow keys\n" "Press to show activity details\n" + "Press to toggle free arrow movement\n" "Press to show/hide this help\n" "Press to exit\n" ) @@ -290,7 +303,7 @@ def _update_help_overlay(self, screen): del self.help_overlay self.help_overlay = None - def _move_viewscreen(self): + def _move_viewscreen_to_activity(self): """Move viewscreen to include selected activity.""" if self.activity_start.x - 1 < self.x_pos: self.x_pos = max(self.activity_start.x - 1, 0) @@ -314,17 +327,31 @@ def _loop(self, screen): # handle keypress if input_char == curses.KEY_DOWN or chr(input_char) == "k": - self._change_layer(1) + if self.free_move: + self.y_pos = min(self.y_pos + 1, self.content_max_y - self.rows - 1) + else: + self._change_layer(1) elif input_char == curses.KEY_UP or chr(input_char) == "i": - self._change_layer(-1) + if self.free_move: + self.y_pos = max(self.y_pos - 1, 0) + else: + self._change_layer(-1) elif input_char == curses.KEY_RIGHT or chr(input_char) == "l": - self._change_layer_position(1) + if self.free_move: + self.x_pos = min(self.x_pos + 1, self.content_max_x - self.cols - 1) + else: + self._change_layer_position(1) elif input_char == curses.KEY_LEFT or chr(input_char) == "j": - self._change_layer_position(-1) + if self.free_move: + self.x_pos = max(self.x_pos - 1, 0) + else: + self._change_layer_position(-1) elif input_char == curses.KEY_ENTER or input_char == 10 or input_char == 13: self._update_activity_overlay(screen) elif chr(input_char) == "h": self._update_help_overlay(screen) + elif chr(input_char) == "f": + self.free_move = not self.free_move elif input_char < 256 and chr(input_char) == "q": running = False @@ -332,7 +359,8 @@ def _loop(self, screen): self._select_activity() self._blink_text(self.content_pad, self.activity_start, self.activity_end, bold=True) - self._move_viewscreen() + if not self.free_move: + self._move_viewscreen_to_activity() self._refresh(screen) diff --git a/renku/ui/cli/workflow.py b/renku/ui/cli/workflow.py index 0e7f29bbe9..4276ce70a3 100644 --- a/renku/ui/cli/workflow.py +++ b/renku/ui/cli/workflow.py @@ -1193,7 +1193,7 @@ def execute( ) -@workflow.command(no_args_is_help=True) +@workflow.command() @click.option( "--from", "sources", @@ -1267,7 +1267,7 @@ def visualize(sources, columns, exclude_files, ascii, revision, format, interact max_width = max(node[1].x for layer in navigation_data for node in layer) tty_size = shutil.get_terminal_size(fallback=(120, 120)) - if no_pager or not sys.stdout.isatty() or os.system(f"less 2>{os.devnull}") != 0: + if no_pager or not sys.stdout.isatty() or os.system(f"less 2>{os.devnull}") != 0: # nosec use_pager = False elif pager: use_pager = True From 8844dffbb6309199ce30c16484718ea81366b738 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Wed, 15 Feb 2023 15:39:34 +0100 Subject: [PATCH 06/13] chore: fix coveralls comments (#3320) --- .github/workflows/test_deploy.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_deploy.yml b/.github/workflows/test_deploy.yml index c0be5ca0cc..3d002c8127 100644 --- a/.github/workflows/test_deploy.yml +++ b/.github/workflows/test_deploy.yml @@ -3,9 +3,13 @@ name: Test, Integration Tests and Deploy on: push: branches: - - "**" + - master + - develop tags: - "v*.*.*" + pull_request: + types: [opened, reopened, synchronize] + env: DEPENDENCY_CACHE_PREFIX: "v2" NETWORK_CACHE_PREFIX: "v1" From 4a5206971fb8212a399d06132fe811d948edb96f Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Thu, 16 Feb 2023 10:22:31 +0100 Subject: [PATCH 07/13] fix(svc): properly use Redis sentinel (#3319) --- renku/ui/cli/service.py | 7 +++--- renku/ui/service/cache/base.py | 39 ++++++++++++++++++++++---------- renku/ui/service/cache/config.py | 8 ------- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/renku/ui/cli/service.py b/renku/ui/cli/service.py index 51c2f8a49e..e1201ecaf2 100644 --- a/renku/ui/cli/service.py +++ b/renku/ui/cli/service.py @@ -176,7 +176,7 @@ def read_logs(log_file, follow=True, output_all=False): @click.pass_context def service(ctx, env): """Manage service components.""" - import redis # noqa: F401 + import redis import rq # noqa: F401 from dotenv import load_dotenv @@ -196,10 +196,9 @@ def service(ctx, env): ctx.exit(1) - except redis.exceptions.ConnectionError: + except redis.exceptions.ConnectionError as e: # NOTE: Cannot connect to the service dependencies, ie. Redis. - - click.echo(ERROR + "Cannot connect to Redis") + click.echo(ERROR + f"Cannot connect to Redis: {e}") ctx.exit(1) diff --git a/renku/ui/service/cache/base.py b/renku/ui/service/cache/base.py index 99c02d767b..149ffa4454 100644 --- a/renku/ui/service/cache/base.py +++ b/renku/ui/service/cache/base.py @@ -18,28 +18,43 @@ """Renku service cache management.""" import json import os +from typing import Any, Dict import redis from redis import RedisError from walrus import Database -from renku.ui.service.cache.config import REDIS_DATABASE, REDIS_HOST, REDIS_NAMESPACE, REDIS_PASSWORD, REDIS_PORT +from renku.ui.service.cache.config import ( + REDIS_DATABASE, + REDIS_HOST, + REDIS_IS_SENTINEL, + REDIS_MASTER_SET, + REDIS_NAMESPACE, + REDIS_PASSWORD, + REDIS_PORT, +) + +_config: Dict[str, Any] = { + "db": REDIS_DATABASE, + "password": REDIS_PASSWORD, + "retry_on_timeout": True, + "health_check_interval": int(os.getenv("CACHE_HEALTH_CHECK_INTERVAL", 60)), +} + +if REDIS_IS_SENTINEL: + _sentinel = redis.Sentinel([(REDIS_HOST, REDIS_PORT)], sentinel_kwargs={"password": REDIS_PASSWORD}) + _cache = _sentinel.master_for(REDIS_MASTER_SET, **_config) + _model_db = Database(connection_pool=_cache.connection_pool, **_config) +else: + _cache = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, **_config) # type: ignore + _model_db = Database(host=REDIS_HOST, port=REDIS_PORT, **_config) class BaseCache: """Cache management.""" - config_ = { - "host": REDIS_HOST, - "port": REDIS_PORT, - "db": REDIS_DATABASE, - "password": REDIS_PASSWORD, - "retry_on_timeout": True, - "health_check_interval": int(os.getenv("CACHE_HEALTH_CHECK_INTERVAL", 60)), - } - - cache = redis.Redis(**config_) # type: ignore - model_db = Database(**config_) + cache: redis.Redis = _cache + model_db: Database = _model_db namespace = REDIS_NAMESPACE def set_record(self, name, key, value): diff --git a/renku/ui/service/cache/config.py b/renku/ui/service/cache/config.py index 3576d01b5f..e1a6623e98 100644 --- a/renku/ui/service/cache/config.py +++ b/renku/ui/service/cache/config.py @@ -27,11 +27,3 @@ REDIS_IS_SENTINEL = os.environ.get("REDIS_IS_SENTINEL", "") == "true" REDIS_MASTER_SET = os.environ.get("REDIS_MASTER_SET", "mymaster") -if REDIS_IS_SENTINEL: - from redis.sentinel import Sentinel - - sentinel = Sentinel( - [(REDIS_HOST, REDIS_PORT)], - sentinel_kwargs={"password": REDIS_PASSWORD}, - ) - REDIS_HOST, REDIS_PORT = sentinel.discover_master(REDIS_MASTER_SET) From 38aa53c3fa2a15976ff1cce68b5a21ca24df2078 Mon Sep 17 00:00:00 2001 From: Mohammad Alisafaee Date: Tue, 21 Feb 2023 16:07:28 +0100 Subject: [PATCH 08/13] fix(core): metadata v10 migration fixes (#3326) --- renku/command/checks/__init__.py | 7 +- renku/command/checks/activities.py | 58 +++++++++++ renku/command/checks/workflow.py | 8 +- .../migration/m_0009__new_metadata_storage.py | 23 ++--- .../core/migration/m_0010__metadata_fixes.py | 95 +++++++++++++++---- renku/core/util/datetime8601.py | 5 +- renku/core/util/git.py | 6 +- renku/data/shacl_shape.json | 1 - renku/domain_model/workflow/plan.py | 5 +- .../gateway/database_gateway.py | 1 + tests/core/metadata/test_project_gateway.py | 5 +- tests/core/models/test_activity.py | 6 +- tests/core/test_plan.py | 4 +- tests/utils.py | 7 +- 14 files changed, 173 insertions(+), 58 deletions(-) diff --git a/renku/command/checks/__init__.py b/renku/command/checks/__init__.py index 2c60b50a53..3b6bf21fcf 100644 --- a/renku/command/checks/__init__.py +++ b/renku/command/checks/__init__.py @@ -17,7 +17,7 @@ # limitations under the License. """Define repository checks for :program:`renku doctor`.""" -from .activities import check_migrated_activity_ids +from .activities import check_activity_dates, check_migrated_activity_ids from .datasets import ( check_dataset_files_outside_datadir, check_dataset_old_metadata_location, @@ -30,7 +30,7 @@ from .project import check_project_id_group from .storage import check_lfs_info from .validate_shacl import check_datasets_structure, check_project_structure -from .workflow import check_activity_catalog, check_modification_date +from .workflow import check_activity_catalog, check_plan_modification_date # Checks will be executed in the order as they are listed in __all__. They are mostly used in ``doctor`` command to # inspect broken things. The order of operations matters when fixing issues, so, don't sort this list. @@ -48,5 +48,6 @@ "check_missing_files", "check_project_id_group", "check_project_structure", - "check_modification_date", + "check_plan_modification_date", + "check_activity_dates", ) diff --git a/renku/command/checks/activities.py b/renku/command/checks/activities.py index be7e9ebaee..3a6f6b3a5e 100644 --- a/renku/command/checks/activities.py +++ b/renku/command/checks/activities.py @@ -71,3 +71,61 @@ def check_migrated_activity_ids(fix, activity_gateway: IActivityGateway, **_): ) return False, problems + + +@inject.autoparams("activity_gateway") +def check_activity_dates(fix, activity_gateway: IActivityGateway, **_): + """Check activities have correct start/end/delete dates. + + Args: + fix(bool): Whether to fix found issues. + activity_gateway(IActivityGateway): Injected ActivityGateway. + _: keyword arguments. + + Returns: + Tuple[bool, Optional[str]]: Tuple of whether there are activities with invalid dates a string of the problem. + """ + invalid_activities = [] + + for activity in activity_gateway.get_all_activities(include_deleted=True): + plan = activity.association.plan + if ( + activity.started_at_time < plan.date_created + or activity.ended_at_time < activity.started_at_time + or (activity.invalidated_at and activity.invalidated_at < activity.ended_at_time) + ): + invalid_activities.append(activity) + + if not invalid_activities: + return True, None + if not fix: + ids = [a.id for a in invalid_activities] + message = ( + WARNING + + "The following activity have incorrect start, end, or delete date (use 'renku doctor --fix' to fix them):" + + "\n\t" + + "\n\t".join(ids) + ) + return False, message + + fix_activity_dates(activities=invalid_activities) + project_context.database.commit() + communication.info("Activity dates were fixed") + + return True, None + + +def fix_activity_dates(activities): + """Fix activities' start/end/delete dates.""" + for activity in activities: + plan = activity.association.plan + activity.unfreeze() + if activity.started_at_time < plan.date_created: + activity.started_at_time = plan.date_created + + if activity.ended_at_time < activity.started_at_time: + activity.ended_at_time = activity.started_at_time + + if activity.invalidated_at and activity.invalidated_at < activity.ended_at_time: + activity.invalidated_at = activity.ended_at_time + activity.freeze() diff --git a/renku/command/checks/workflow.py b/renku/command/checks/workflow.py index 2e3605877d..4202defa7b 100644 --- a/renku/command/checks/workflow.py +++ b/renku/command/checks/workflow.py @@ -16,7 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Checks needed to determine integrity of workflows.""" - +from datetime import timedelta from typing import List, Optional, Tuple, cast from renku.command.command_builder import inject @@ -64,7 +64,7 @@ def check_activity_catalog(fix, force, **_) -> Tuple[bool, Optional[str]]: @inject.autoparams("plan_gateway") -def check_modification_date(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, Optional[str]]: +def check_plan_modification_date(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, Optional[str]]: """Check if all plans have modification date set for them. Args: @@ -88,7 +88,7 @@ def check_modification_date(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, ids = [plan.id for plan in to_be_processed] message = ( WARNING - + "The following workflows have incorrect modification date (use 'renku doctor --fix' to fix them).:\n\t" + + "The following workflows have incorrect modification date (use 'renku doctor --fix' to fix them):\n\t" + "\n\t".join(ids) ) return False, message @@ -124,4 +124,6 @@ def fix_plan_dates(plans: List[AbstractPlan], plan_gateway): plan.unfreeze() plan.date_modified = plan.date_created plan.date_created = creation_date + if plan.date_removed and plan.date_removed < plan.date_created: + plan.date_removed = plan.date_created + timedelta(seconds=1) plan.freeze() diff --git a/renku/core/migration/m_0009__new_metadata_storage.py b/renku/core/migration/m_0009__new_metadata_storage.py index 10651b503f..e65db3ced6 100644 --- a/renku/core/migration/m_0009__new_metadata_storage.py +++ b/renku/core/migration/m_0009__new_metadata_storage.py @@ -655,19 +655,16 @@ def _process_datasets( for dataset in datasets: dataset, tags = convert_dataset(dataset=dataset, revision=revision) - if is_last_commit: - datasets_provenance.update_during_migration( - dataset, - commit_sha=revision, - date=date, - tags=tags, - replace=True, - preserve_identifiers=preserve_identifiers, - ) - else: - datasets_provenance.update_during_migration( - dataset, commit_sha=revision, date=date, tags=tags, preserve_identifiers=preserve_identifiers - ) + + datasets_provenance.update_during_migration( + dataset, + commit_sha=revision, + date=date, + tags=tags, + replace=True if is_last_commit else False, + preserve_identifiers=preserve_identifiers, + ) + for dataset in deleted_datasets: dataset, _ = convert_dataset(dataset=dataset, revision=revision) datasets_provenance.update_during_migration( diff --git a/renku/core/migration/m_0010__metadata_fixes.py b/renku/core/migration/m_0010__metadata_fixes.py index 150a181fc4..e98f8457e3 100644 --- a/renku/core/migration/m_0010__metadata_fixes.py +++ b/renku/core/migration/m_0010__metadata_fixes.py @@ -16,6 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Various metadata migrations for v10.""" + import io import json import os @@ -26,6 +27,7 @@ import zstandard as zstd +from renku.command.checks.activities import fix_activity_dates from renku.command.checks.workflow import fix_plan_dates from renku.command.command_builder import inject from renku.core.interface.activity_gateway import IActivityGateway @@ -53,9 +55,12 @@ def migrate(migration_context: MigrationContext): if MigrationType.WORKFLOWS in migration_context.options.type: migrate_activity_ids() fix_plan_times() + fix_activity_times() migrate_remote_entity_ids() fix_dataset_date_modified() + fix_dataset_image_ids() + fix_removed_plans() # NOTE: Rebuild all workflow catalogs since ids and times have changed communication.echo("Rebuilding workflow metadata") @@ -76,8 +81,8 @@ def migrate_old_metadata_namespaces(): header = int.from_bytes(file.read(4), "little") file.seek(0) if header == zstd.MAGIC_NUMBER: - with decompressor.stream_reader(file) as zfile: - data = json.load(zfile) + with decompressor.stream_reader(file) as compressed_file: + data = json.load(compressed_file) compressed = True else: data = json.load(file) @@ -99,7 +104,7 @@ def migrate_old_metadata_namespaces(): def nested_update(data: Dict[str, Any], target_key: str, transforms: List[Tuple[str, str]]) -> None: - """Update a key's value based on tranformations (from, to) in a deeply nested dictionary.""" + """Update a key's value based on transformations (from, to) in a deeply nested dictionary.""" for k in list(data.keys()): value = data[k] if isinstance(value, str) and k == target_key: @@ -232,19 +237,10 @@ def migrate_project_template_data(project_gateway: IProjectGateway): project_context.database.commit() -@inject.autoparams("activity_gateway", "plan_gateway") -def fix_plan_times(activity_gateway: IActivityGateway, plan_gateway: IPlanGateway): +@inject.autoparams("plan_gateway") +def fix_plan_times(plan_gateway: IPlanGateway): """Add timezone to plan invalidations.""" - database = project_context.database - plans: List[AbstractPlan] = plan_gateway.get_all_plans() - all_activities = activity_gateway.get_all_activities() - activity_map: Dict[str, Activity] = {} - - for activity in all_activities: - plan_id = activity.association.plan.id - if plan_id not in activity_map or activity.started_at_time < activity_map[plan_id].started_at_time: - activity_map[plan_id] = activity for plan in plans: plan.unfreeze() @@ -255,24 +251,41 @@ def fix_plan_times(activity_gateway: IActivityGateway, plan_gateway: IPlanGatewa plan.date_removed = None if plan.date_removed is not None: - if plan.date_removed < plan.date_created: - # NOTE: Fix invalidation times set before creation date on plans - plan.date_removed = plan.date_created if plan.date_removed.tzinfo is None: # NOTE: There was a bug that caused date_removed to be set without timezone (as UTC time) # so we patch in the timezone here plan.date_removed = plan.date_removed.replace(microsecond=0).astimezone(timezone.utc) - if plan.id in activity_map and plan.date_created > activity_map[plan.id].started_at_time: - plan.date_created = activity_map[plan.id].started_at_time + if plan.date_removed < plan.date_created: + # NOTE: Fix invalidation times set before creation date on plans + plan.date_removed = plan.date_created plan.freeze() fix_plan_dates(plans=plans, plan_gateway=plan_gateway) - database.commit() + project_context.database.commit() + + +@inject.autoparams("activity_gateway") +def fix_activity_times(activity_gateway: IActivityGateway): + """Make sure activities have valid start/end/delete dates.""" + fix_activity_dates(activities=activity_gateway.get_all_activities(include_deleted=True)) + project_context.database.commit() @inject.autoparams("dataset_gateway") def fix_dataset_date_modified(dataset_gateway: IDatasetGateway): """Change date_created and date_modified to have correct semantics.""" + + def fix_creation_date(dataset): + """Check creation date to make sure that it's after project's creation date.""" + if dataset.date_created and dataset.date_created < project_context.project.date_created: + try: + dataset.date_created = min([f.date_added for f in dataset.files]) + except (ValueError, TypeError): + dataset.date_created = project_context.project.date_created + else: + if dataset.date_created < project_context.project.date_created: + dataset.date_created = project_context.project.date_created + tails = dataset_gateway.get_provenance_tails() for dataset_tail in tails: @@ -281,6 +294,7 @@ def fix_dataset_date_modified(dataset_gateway: IDatasetGateway): previous_modification_date = local_now() while dataset.derived_from is not None: + fix_creation_date(dataset) modification_date = dataset.date_removed or dataset.date_created if modification_date is not None: @@ -294,8 +308,9 @@ def fix_dataset_date_modified(dataset_gateway: IDatasetGateway): found_datasets.append(dataset) dataset = dataset_gateway.get_by_id(dataset.derived_from.value) + fix_creation_date(dataset) # NOTE: first dataset in chain - modification_date = dataset.date_created or dataset.date_published + modification_date = dataset.date_published or dataset.date_created if modification_date is not None: dataset.unfreeze() dataset.date_modified = modification_date @@ -308,3 +323,41 @@ def fix_dataset_date_modified(dataset_gateway: IDatasetGateway): child.freeze() project_context.database.commit() + + +@inject.autoparams("dataset_gateway") +def fix_dataset_image_ids(dataset_gateway: IDatasetGateway): + """Remove dashes from dataset image IDs.""" + for dataset in dataset_gateway.get_provenance_tails(): + while True: + if dataset.images: + for image in dataset.images: + image.id = image.id.replace("-", "") + + dataset._p_changed = True + + if not dataset.derived_from: + break + + dataset = dataset_gateway.get_by_id(dataset.derived_from.value) + + project_context.database.commit() + + +@inject.autoparams("plan_gateway") +def fix_removed_plans(plan_gateway: IPlanGateway): + """Create a derivative if a removed plan doesn't have one.""" + plans: List[AbstractPlan] = plan_gateway.get_all_plans() + + for plan in plans: + if plan.date_removed and plan.derived_from is None: + derived_plan = plan.derive() + derived_plan.date_modified = plan.date_modified + derived_plan.delete(when=plan.date_removed) + plan_gateway.add(derived_plan) + + plan.unfreeze() + plan.date_removed = None + plan.freeze() + + project_context.database.commit() diff --git a/renku/core/util/datetime8601.py b/renku/core/util/datetime8601.py index b6721f6272..dca65475b6 100644 --- a/renku/core/util/datetime8601.py +++ b/renku/core/util/datetime8601.py @@ -75,6 +75,7 @@ def _set_to_local_timezone(value): return value.replace(tzinfo=local_tz) -def local_now() -> datetime: +def local_now(remove_microseconds: bool = True) -> datetime: """Return current datetime in local timezone.""" - return datetime.now(timezone.utc).replace(microsecond=0).astimezone() + now = datetime.now(timezone.utc).astimezone() + return now.replace(microsecond=0) if remove_microseconds else now diff --git a/renku/core/util/git.py b/renku/core/util/git.py index b45c220627..2a18a6da65 100644 --- a/renku/core/util/git.py +++ b/renku/core/util/git.py @@ -379,7 +379,7 @@ def get_entity_from_revision( Entity: The Entity for the given path and revision. """ - from renku.domain_model.entity import Collection, Entity + from renku.domain_model.entity import NON_EXISTING_ENTITY_CHECKSUM, Collection, Entity def get_directory_members(absolute_path: Path) -> List[Entity]: """Return first-level files/directories in a directory.""" @@ -410,7 +410,9 @@ def get_directory_members(absolute_path: Path) -> List[Entity]: checksum = repository.get_object_hash(revision=revision, path=path) # NOTE: If object was not found at a revision it's either removed or exists in a different revision; keep the # entity and use revision as checksum - checksum = checksum or revision or "HEAD" + if isinstance(revision, str) and revision == "HEAD": + revision = repository.head.commit.hexsha + checksum = checksum or revision or NON_EXISTING_ENTITY_CHECKSUM id = Entity.generate_id(checksum=checksum, path=path) absolute_path = repository.path / path diff --git a/renku/data/shacl_shape.json b/renku/data/shacl_shape.json index 33dd9ec47a..f52cc3f4fb 100644 --- a/renku/data/shacl_shape.json +++ b/renku/data/shacl_shape.json @@ -1635,7 +1635,6 @@ "datatype": { "@id": "xsd:string" }, - "minCount": 1, "maxCount": 1 }, { diff --git a/renku/domain_model/workflow/plan.py b/renku/domain_model/workflow/plan.py index 851585c038..59afe50fe4 100644 --- a/renku/domain_model/workflow/plan.py +++ b/renku/domain_model/workflow/plan.py @@ -153,14 +153,14 @@ def is_derivation(self) -> bool: """Return if an ``AbstractPlan`` has correct derived_from.""" raise NotImplementedError() - def delete(self): + def delete(self, when: datetime = local_now()): """Mark a plan as deleted. NOTE: Don't call this function for deleting plans since it doesn't delete the whole plan derivatives chain. Use renku.core.workflow.plan::remove_plan instead. """ self.unfreeze() - self.date_removed = local_now() + self.date_removed = when self.freeze() @@ -323,6 +323,7 @@ def derive(self, creator: Optional[Person] = None) -> "Plan": derived.keywords = copy.deepcopy(self.keywords) derived.outputs = self.outputs.copy() derived.success_codes = self.success_codes.copy() + derived.creators = self.creators.copy() derived.assign_new_id() if creator and hasattr(creator, "email") and not any(c for c in self.creators if c.email == creator.email): diff --git a/renku/infrastructure/gateway/database_gateway.py b/renku/infrastructure/gateway/database_gateway.py index cb17b04cb3..18670e6aa6 100644 --- a/renku/infrastructure/gateway/database_gateway.py +++ b/renku/infrastructure/gateway/database_gateway.py @@ -85,6 +85,7 @@ def load_downstream_relations(token, catalog, cache): def initialize_database(database): """Initialize an empty database with all required metadata.""" + # NOTE: A list of existing and removed activities database.add_index(name="activities", object_type=Activity, attribute="id") database.add_root_object(name="activities-by-usage", obj=RenkuOOBTree()) database.add_root_object(name="activities-by-generation", obj=RenkuOOBTree()) diff --git a/tests/core/metadata/test_project_gateway.py b/tests/core/metadata/test_project_gateway.py index ac00cfac07..7e7f156198 100644 --- a/tests/core/metadata/test_project_gateway.py +++ b/tests/core/metadata/test_project_gateway.py @@ -17,8 +17,7 @@ # limitations under the License. """Test project database gateways.""" -from datetime import datetime - +from renku.core.util.datetime8601 import local_now from renku.domain_model.project import Project from renku.domain_model.provenance.agent import Person from renku.infrastructure.gateway.project_gateway import ProjectGateway @@ -32,7 +31,7 @@ def test_project_gateway_update(project_with_injection, monkeypatch): creator=Person( id=Person.generate_id("test@test.com", "Test tester"), email="test@test.com", name="Test tester" ), - date_created=datetime.utcnow(), + date_created=local_now(), ) project_gateway = ProjectGateway() diff --git a/tests/core/models/test_activity.py b/tests/core/models/test_activity.py index edca7f999d..e116f7d476 100644 --- a/tests/core/models/test_activity.py +++ b/tests/core/models/test_activity.py @@ -17,10 +17,10 @@ # limitations under the License. """Test Activity.""" -from datetime import datetime from unittest.mock import MagicMock from uuid import uuid4 +from renku.core.util.datetime8601 import local_now from renku.domain_model.entity import Entity from renku.domain_model.provenance.activity import Activity from renku.domain_model.provenance.agent import Person @@ -77,8 +77,8 @@ def get_git_user_mock(repository): plan, repository=project_with_injection.repository, project_gateway=project_gateway, - started_at_time=datetime.utcnow(), - ended_at_time=datetime.utcnow(), + started_at_time=local_now(remove_microseconds=False), + ended_at_time=local_now(remove_microseconds=False), annotations=[], ) diff --git a/tests/core/test_plan.py b/tests/core/test_plan.py index efca88f1a3..7a63463293 100644 --- a/tests/core/test_plan.py +++ b/tests/core/test_plan.py @@ -20,7 +20,7 @@ import pytest -from renku.command.checks import check_modification_date +from renku.command.checks import check_plan_modification_date from renku.core import errors from renku.core.workflow.plan import ( get_activities, @@ -176,7 +176,7 @@ def test_modification_date_fix(project_with_injection): del plan.date_modified unrelated.date_modified = None - check_modification_date(fix=True) + check_plan_modification_date(fix=True) assert dummy_date == plan.date_modified assert unrelated.date_created == unrelated.date_modified diff --git a/tests/utils.py b/tests/utils.py index a1ebd5e1b4..f8907160c7 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -22,7 +22,7 @@ import traceback import uuid from contextlib import contextmanager -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from functools import wraps from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, Iterator, List, Optional, Tuple, Type, Union @@ -36,6 +36,7 @@ from renku.command.command_builder.command import inject, replace_injection from renku.core.interface.dataset_gateway import IDatasetGateway +from renku.core.util.datetime8601 import local_now from renku.domain_model.dataset import Dataset from renku.domain_model.project_context import project_context @@ -308,7 +309,7 @@ def create_dummy_activity( assert isinstance(plan, str) plan = Plan(id=Plan.generate_id(), name=plan, command=plan) - ended_at_time = ended_at_time or datetime.utcnow() + ended_at_time = ended_at_time or local_now(remove_microseconds=False) empty_checksum = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" # Git hash of an empty string/file activity_id = id or Activity.generate_id(uuid=None if index is None else str(index)) @@ -388,7 +389,7 @@ def create_dummy_plan( plan = Plan( command=command, - date_created=date_created or datetime(2022, 5, 20, 0, 42, 0), + date_created=date_created or datetime(2022, 5, 20, 0, 42, 0, tzinfo=timezone.utc), description=description, id=id, inputs=[], From c024650c96beef6d1ae2153b18b5b142171f2d80 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Fri, 24 Feb 2023 15:45:53 +0100 Subject: [PATCH 09/13] feat(core): add support for ssh connections into sessions (#3318) --- renku/command/format/session.py | 4 + renku/command/migrate.py | 6 + renku/command/session.py | 9 +- renku/command/view_model/project.py | 4 + renku/core/errors.py | 22 +++ renku/core/init.py | 2 + renku/core/login.py | 7 +- renku/core/session/docker.py | 35 +++- renku/core/session/renkulab.py | 172 +++++++++++++++- renku/core/session/session.py | 156 ++++++++++++--- renku/core/session/utils.py | 5 + renku/core/storage.py | 6 +- renku/core/template/template.py | 8 +- renku/core/util/git.py | 4 +- renku/core/util/ssh.py | 183 ++++++++++++++++++ renku/domain_model/project.py | 1 + renku/domain_model/project_context.py | 9 +- renku/domain_model/session.py | 48 +++-- renku/domain_model/template.py | 3 + renku/infrastructure/repository.py | 24 +++ renku/ui/cli/config.py | 10 +- renku/ui/cli/login.py | 4 +- renku/ui/cli/project.py | 3 + renku/ui/cli/session.py | 89 ++++++++- renku/ui/cli/utils/plugins.py | 2 +- .../controllers/cache_migrations_check.py | 8 +- .../controllers/templates_create_project.py | 1 + renku/ui/service/serializers/cache.py | 4 + renku/ui/service/serializers/templates.py | 3 + renku/ui/service/serializers/v0_9/__init__.py | 18 -- renku/ui/service/serializers/v0_9/cache.py | 46 ----- .../serializers/{v1_0 => v1}/__init__.py | 0 renku/ui/service/serializers/v1/cache.py | 105 ++++++++++ renku/ui/service/serializers/v1/templates.py | 48 +++++ renku/ui/service/serializers/v1_0/cache.py | 38 ---- renku/ui/service/views/api_versions.py | 16 +- renku/ui/service/views/cache.py | 12 +- renku/ui/service/views/graph.py | 4 +- renku/ui/service/views/project.py | 12 +- renku/ui/service/views/templates.py | 8 +- renku/ui/service/views/v0_9/__init__.py | 18 -- renku/ui/service/views/v0_9/cache.py | 61 ------ .../ui/service/views/{v1_0 => v1}/__init__.py | 0 renku/ui/service/views/v1/cache.py | 103 ++++++++++ renku/ui/service/views/v1/templates.py | 76 ++++++++ renku/ui/service/views/v1_0/cache.py | 63 ------ renku/ui/service/views/workflow_plans.py | 4 +- tests/cli/fixtures/cli_projects.py | 4 +- tests/cli/test_login.py | 4 +- tests/cli/test_migrate.py | 1 + tests/cli/test_session.py | 37 +++- tests/core/plugins/test_session.py | 140 +++++++++++++- tests/fixtures/communication.py | 9 + tests/fixtures/repository.py | 4 +- tests/fixtures/session.py | 21 +- tests/fixtures/templates.py | 1 + .../service/views/test_api_versions_views.py | 14 +- tests/service/views/test_cache_views.py | 2 + tests/service/views/test_version_views.py | 8 +- .../views/v0_9/test_cache_views_0_9.py | 87 --------- 60 files changed, 1327 insertions(+), 469 deletions(-) create mode 100644 renku/core/util/ssh.py delete mode 100644 renku/ui/service/serializers/v0_9/__init__.py delete mode 100644 renku/ui/service/serializers/v0_9/cache.py rename renku/ui/service/serializers/{v1_0 => v1}/__init__.py (100%) create mode 100644 renku/ui/service/serializers/v1/cache.py create mode 100644 renku/ui/service/serializers/v1/templates.py delete mode 100644 renku/ui/service/serializers/v1_0/cache.py delete mode 100644 renku/ui/service/views/v0_9/__init__.py delete mode 100644 renku/ui/service/views/v0_9/cache.py rename renku/ui/service/views/{v1_0 => v1}/__init__.py (100%) create mode 100644 renku/ui/service/views/v1/cache.py create mode 100644 renku/ui/service/views/v1/templates.py delete mode 100644 renku/ui/service/views/v1_0/cache.py delete mode 100644 tests/service/views/v0_9/test_cache_views_0_9.py diff --git a/renku/command/format/session.py b/renku/command/format/session.py index cf36bf5460..5f850a7b81 100644 --- a/renku/command/format/session.py +++ b/renku/command/format/session.py @@ -25,6 +25,9 @@ def tabular(sessions, *, columns=None): if not columns: columns = "id,status,url" + if any(s.ssh_enabled for s in sessions): + columns += ",ssh" + return tabulate(collection=sessions, columns=columns, columns_mapping=SESSION_COLUMNS) @@ -35,4 +38,5 @@ def tabular(sessions, *, columns=None): "id": ("id", "id"), "status": ("status", "status"), "url": ("url", "url"), + "ssh": ("ssh_enabled", "SSH enabled"), } diff --git a/renku/command/migrate.py b/renku/command/migrate.py index 9ac360c944..d2a56c8a04 100644 --- a/renku/command/migrate.py +++ b/renku/command/migrate.py @@ -88,6 +88,7 @@ def _template_migration_check(): Returns: Dictionary of template migration status. """ + from renku.core.config import get_value from renku.core.template.usecase import check_for_template_update try: @@ -95,11 +96,15 @@ def _template_migration_check(): template_source = project.template_metadata.template_source template_ref = project.template_metadata.template_ref template_id = project.template_metadata.template_id + ssh_supported = project.template_metadata.ssh_supported except (ValueError, AttributeError): project = None template_source = None template_ref = None template_id = None + ssh_supported = False + + ssh_supported = get_value("renku", "ssh_supported") == "true" or ssh_supported update_available, update_allowed, current_version, new_version = check_for_template_update(project) @@ -111,6 +116,7 @@ def _template_migration_check(): "template_source": template_source, "template_ref": template_ref, "template_id": template_id, + "ssh_supported": ssh_supported, } diff --git a/renku/command/session.py b/renku/command/session.py index 992f15748f..3f7fea52de 100644 --- a/renku/command/session.py +++ b/renku/command/session.py @@ -19,7 +19,7 @@ from renku.command.command_builder.command import Command -from renku.core.session.session import session_list, session_open, session_start, session_stop +from renku.core.session.session import session_list, session_open, session_start, session_stop, ssh_setup def session_list_command(): @@ -29,7 +29,7 @@ def session_list_command(): def session_start_command(): """Start an interactive session.""" - return Command().command(session_start) + return Command().command(session_start).with_database() def session_stop_command(): @@ -40,3 +40,8 @@ def session_stop_command(): def session_open_command(): """Open a running interactive session.""" return Command().command(session_open) + + +def ssh_setup_command(): + """Setup SSH keys for SSH connections to sessions.""" + return Command().command(ssh_setup) diff --git a/renku/command/view_model/project.py b/renku/command/view_model/project.py index 9f48296321..d0c0704999 100644 --- a/renku/command/view_model/project.py +++ b/renku/command/view_model/project.py @@ -21,6 +21,7 @@ from datetime import datetime from typing import List, Optional +from renku.core.config import get_value from renku.domain_model.project import Project from renku.domain_model.provenance.agent import Person @@ -39,6 +40,7 @@ def __init__( annotations: Optional[str], template_info: str, keywords: Optional[List[str]], + ssh_supported: bool = False, ): self.id = id self.name = name @@ -52,6 +54,7 @@ def __init__( self.template_info = template_info self.keywords = keywords self.keywords_str = ", ".join(keywords) if keywords else "" + self.ssh_supported = ssh_supported @classmethod def from_project(cls, project: Project): @@ -88,4 +91,5 @@ def from_project(cls, project: Project): else None, template_info=template_info, keywords=project.keywords, + ssh_supported=get_value("renku", "ssh_supported") == "true" or project.template_metadata.ssh_supported, ) diff --git a/renku/core/errors.py b/renku/core/errors.py index 6050923932..bcea0b8f78 100644 --- a/renku/core/errors.py +++ b/renku/core/errors.py @@ -110,6 +110,10 @@ class AuthenticationError(RenkuException): """Raise when there is a problem with authentication.""" +class KeyNotFoundError(RenkuException): + """Raise when an SSH private or public key couldn't be found.""" + + class DirtyRepository(RenkuException): """Raise when trying to work with dirty repository.""" @@ -526,6 +530,24 @@ def __init__(self): super(NodeNotFoundError, self).__init__(msg) +class SSHNotFoundError(RenkuException): + """Raised when SSH client is not installed on the system.""" + + def __init__(self): + """Build a custom message.""" + msg = "SSH client (ssh) could not be found on this system" + super(SSHNotFoundError, self).__init__(msg) + + +class SSHNotSetupError(RenkuException): + """Raised when SSH client is not installed on the system.""" + + def __init__(self): + """Build a custom message.""" + msg = "SSH is not set up correctly, use 'renku session ssh-setup' to set it up." + super(SSHNotSetupError, self).__init__(msg) + + class ObjectNotFoundError(RenkuException): """Raised when an object is not found in the storage.""" diff --git a/renku/core/init.py b/renku/core/init.py index d4c53f074f..4ab5a66282 100644 --- a/renku/core/init.py +++ b/renku/core/init.py @@ -338,6 +338,7 @@ def create_from_template_local( description: Optional[str] = None, keywords: Optional[List[str]] = None, data_dir: Optional[str] = None, + ssh_supported: bool = False, ): """Initialize a new project from a template. @@ -381,6 +382,7 @@ def create_from_template_local( description="", parameters={}, icon="", + ssh_supported=ssh_supported, immutable_files=immutable_template_files or [], allow_update=automated_template_update, source=metadata["__template_source__"], diff --git a/renku/core/login.py b/renku/core/login.py index 5a58857fd8..afca8977c2 100644 --- a/renku/core/login.py +++ b/renku/core/login.py @@ -64,7 +64,8 @@ def login(endpoint: Optional[str], git_login: bool, yes: bool): remote_name, remote_url = remote.name, remote.url if remote_name and remote_url: - if not yes and not get_value("renku", "show_login_warning"): + show_login_warning = get_value("renku", "show_login_warning") + if not yes and (show_login_warning is None or show_login_warning.lower() == "true"): message = ( "Remote URL will be changed. Do you want to continue " "(to disable this warning, pass '--yes' or run 'renku config set show_login_warning False')?" @@ -140,7 +141,8 @@ def login(endpoint: Optional[str], git_login: bool, yes: bool): ) if backup_exists: communication.echo(f"Backup remote '{backup_remote_name}' already exists.") - elif not remote: + + if not remote and not backup_exists: communication.error(f"Cannot create backup remote '{backup_remote_name}' for '{remote_url}'") else: _set_renku_url_for_remote( @@ -182,7 +184,6 @@ def _set_renku_url_for_remote(repository: "Repository", remote_name: str, remote remote_name(str): Name of the remote. remote_url(str): Url of the remote. hostname(str): Hostname. - Raises: errors.GitCommandError: If remote doesn't exist. """ diff --git a/renku/core/session/docker.py b/renku/core/session/docker.py index 0bc9af55ea..a1d6996b8d 100644 --- a/renku/core/session/docker.py +++ b/renku/core/session/docker.py @@ -17,8 +17,9 @@ # limitations under the License. """Docker based interactive session provider.""" +import webbrowser from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, cast from uuid import uuid4 import docker @@ -32,6 +33,9 @@ from renku.domain_model.project_context import project_context from renku.domain_model.session import ISessionProvider, Session +if TYPE_CHECKING: + from renku.core.dataset.providers.models import ProviderParameter + class DockerSessionProvider(ISessionProvider): """A docker based interactive session provider.""" @@ -71,7 +75,8 @@ def _get_jupyter_urls(ports: Dict[str, Any], auth_token: str, jupyter_port: int def _get_docker_containers(self, project_name: str) -> List[docker.models.containers.Container]: return self.docker_client().containers.list(filters={"label": f"renku_project={project_name}"}) - def get_name(self) -> str: + @property + def name(self) -> str: """Return session provider's name.""" return "docker" @@ -109,10 +114,18 @@ def session_provider(self) -> ISessionProvider: """Supported session provider. Returns: - a reference to ``self``. + A reference to ``self``. """ return self + def get_start_parameters(self) -> List["ProviderParameter"]: + """Returns parameters that can be set for session start.""" + return [] + + def get_open_parameters(self) -> List["ProviderParameter"]: + """Returns parameters that can be set for session open.""" + return [] + def session_list(self, project_name: str, config: Optional[Dict[str, Any]]) -> List[Session]: """Lists all the sessions currently running by the given session provider. @@ -140,6 +153,7 @@ def session_start( mem_request: Optional[str] = None, disk_request: Optional[str] = None, gpu_request: Optional[str] = None, + **kwargs, ) -> Tuple[str, str]: """Creates an interactive session. @@ -259,6 +273,21 @@ def session_stop(self, project_name: str, session_name: Optional[str], stop_all: except docker.errors.APIError as error: raise errors.DockerError(error.msg) + def session_open(self, project_name: str, session_name: str, **kwargs) -> bool: + """Open a given interactive session. + + Args: + project_name(str): Renku project name. + session_name(str): The unique id of the interactive session. + """ + url = self.session_url(session_name) + + if not url: + return False + + webbrowser.open(url) + return True + def session_url(self, session_name: str) -> Optional[str]: """Get the URL of the interactive session.""" for c in self.docker_client().containers.list(): diff --git a/renku/core/session/renkulab.py b/renku/core/session/renkulab.py index cd2b669bf5..87cde0cdcd 100644 --- a/renku/core/session/renkulab.py +++ b/renku/core/session/renkulab.py @@ -17,21 +17,28 @@ # limitations under the License. """Docker based interactive session provider.""" +import pty import urllib +import webbrowser from pathlib import Path from time import monotonic, sleep -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from renku.core import errors +from renku.core.config import get_value from renku.core.login import read_renku_token from renku.core.plugin import hookimpl from renku.core.session.utils import get_renku_project_name, get_renku_url from renku.core.util import communication, requests from renku.core.util.git import get_remote from renku.core.util.jwt import is_token_expired +from renku.core.util.ssh import SystemSSHConfig from renku.domain_model.project_context import project_context from renku.domain_model.session import ISessionProvider, Session +if TYPE_CHECKING: + from renku.core.dataset.providers.models import ProviderParameter + class RenkulabSessionProvider(ISessionProvider): """A session provider that uses the notebook service API to launch sessions.""" @@ -134,8 +141,10 @@ def _wait_for_image( "of problems with your Dockerfile." ) - def pre_start_checks(self): + def pre_start_checks(self, ssh: bool = False, **kwargs): """Check if the state of the repository is as expected before starting a session.""" + from renku.core.session.session import ssh_setup + repository = project_context.repository if repository.is_dirty(untracked_files=True): @@ -148,8 +157,46 @@ def pre_start_checks(self): repository.add(all=True) repository.commit("Automated commit by Renku CLI.") - @staticmethod - def _remote_head_hexsha(): + if ssh: + system_config = SystemSSHConfig() + + if not system_config.is_configured: + if communication.confirm( + "Your system is not set up for SSH connections to Renkulab. Would you like to set it up?" + ): + ssh_setup() + else: + raise errors.RenkulabSessionError( + "Can't run ssh session without setting up Renku SSH support. Run without '--ssh' or " + "run 'renku session ssh-setup'." + ) + + system_config.setup_session_keys() + + def _cleanup_ssh_connection_configs( + self, project_name: str, running_sessions: Optional[List[Session]] = None + ) -> None: + """Cleanup leftover SSH connections that aren't valid anymore. + + Args: + project_name(str): Name of the project. + running_sessions(List[Session], optional): List of running sessions to check against, otherwise will be + gotten from the server. + """ + if not running_sessions: + running_sessions = self.session_list("", None, ssh_garbage_collection=False) + + system_config = SystemSSHConfig() + + name = self._project_name_from_full_project_name(project_name) + + session_config_paths = [system_config.session_config_path(name, s.id) for s in running_sessions] + + for path in system_config.renku_ssh_root.glob(f"00-{name}*.conf"): + if path not in session_config_paths: + path.unlink() + + def _remote_head_hexsha(self): remote = get_remote(repository=project_context.repository) if remote is None: @@ -171,7 +218,14 @@ def _send_renku_request(self, req_type: str, *args, **kwargs): ) return res - def get_name(self) -> str: + def _project_name_from_full_project_name(self, project_name: str) -> str: + """Get just project name of project name if in owner/name form.""" + if "/" not in project_name: + return project_name + return project_name.rsplit("/", 1)[1] + + @property + def name(self) -> str: """Return session provider's name.""" return "renkulab" @@ -209,7 +263,25 @@ def session_provider(self) -> ISessionProvider: """ return self - def session_list(self, project_name: str, config: Optional[Dict[str, Any]]) -> List[Session]: + def get_start_parameters(self) -> List["ProviderParameter"]: + """Returns parameters that can be set for session start.""" + from renku.core.dataset.providers.models import ProviderParameter + + return [ + ProviderParameter("ssh", help="Enable ssh connections to the session.", is_flag=True), + ] + + def get_open_parameters(self) -> List["ProviderParameter"]: + """Returns parameters that can be set for session open.""" + from renku.core.dataset.providers.models import ProviderParameter + + return [ + ProviderParameter("ssh", help="Open a remote terminal through SSH.", is_flag=True), + ] + + def session_list( + self, project_name: str, config: Optional[Dict[str, Any]], ssh_garbage_collection: bool = True + ) -> List[Session]: """Lists all the sessions currently running by the given session provider. Returns: @@ -222,14 +294,20 @@ def session_list(self, project_name: str, config: Optional[Dict[str, Any]]) -> L params=self._get_renku_project_name_parts(), ) if sessions_res.status_code == 200: - return [ + system_config = SystemSSHConfig() + name = self._project_name_from_full_project_name(project_name) + sessions = [ Session( session["name"], session.get("status", {}).get("state", "unknown"), self.session_url(session["name"]), + ssh_enabled=system_config.session_config_path(name, session["name"]).exists(), ) for session in sessions_res.json().get("servers", {}).values() ] + if ssh_garbage_collection: + self._cleanup_ssh_connection_configs(project_name, running_sessions=sessions) + return sessions return [] def session_start( @@ -241,12 +319,22 @@ def session_start( mem_request: Optional[str] = None, disk_request: Optional[str] = None, gpu_request: Optional[str] = None, + ssh: bool = False, + **kwargs, ) -> Tuple[str, str]: """Creates an interactive session. Returns: Tuple[str, str]: Provider message and a possible warning message. """ + ssh_supported = ( + get_value("renku", "ssh_supported") == "true" or project_context.project.template_metadata.ssh_supported + ) + if ssh and not ssh_supported: + raise errors.RenkulabSessionError( + "Cannot start session with SSH support because this project doesn't support SSH." + ) + repository = project_context.repository session_commit = repository.head.commit.hexsha @@ -260,6 +348,8 @@ def session_start( abort=True, ) repository.push() + if ssh: + self._cleanup_ssh_connection_configs(project_name) server_options: Dict[str, Union[str, float]] = {} if cpu_request: @@ -285,7 +375,19 @@ def session_start( if res.status_code in [200, 201]: session_name = res.json()["name"] self._wait_for_session_status(session_name, "running") - return f"Session {session_name} successfully started", "" + if ssh: + name = self._project_name_from_full_project_name(project_name) + connection = SystemSSHConfig().setup_session_config(name, session_name) + return ( + f"Session {session_name} successfully started, use 'renku session open --ssh {session_name}'" + f" or 'ssh {connection}' to connect to it", + "", + ) + return ( + f"Session {session_name} successfully started, use 'renku session open {session_name}'" + " to connect to it", + "", + ) raise errors.RenkulabSessionError("Cannot start session via the notebook service because " + res.text) def session_stop(self, project_name: str, session_name: Optional[str], stop_all: bool) -> bool: @@ -307,8 +409,62 @@ def session_stop(self, project_name: str, session_name: Optional[str], stop_all: ) ) self._wait_for_session_status(session_name, "stopping") + + self._cleanup_ssh_connection_configs(project_name) + return all([response.status_code == 204 for response in responses]) if responses else False + def session_open(self, project_name: str, session_name: str, ssh: bool = False, **kwargs) -> bool: + """Open a given interactive session. + + Args: + project_name(str): Renku project name. + session_name(str): The unique id of the interactive session. + ssh(bool): Whether to open an SSH connection or a normal browser interface. + """ + sessions = self.session_list("", None) + system_config = SystemSSHConfig() + name = self._project_name_from_full_project_name(project_name) + ssh_prefix = f"{system_config.renku_host}-{name}-" + + if session_name.startswith(ssh_prefix): + # NOTE: use passed in ssh connection name instead of session id by accident + session_name = session_name.replace(ssh_prefix, "", 1) + + if not any(s.id == session_name for s in sessions): + return False + + if ssh: + ssh_setup = True + if not system_config.is_configured: + raise errors.RenkulabSessionError( + "SSH not set up for session. Run without '--ssh' or " + "run 'renku session ssh-setup' and start the session again." + ) + elif not system_config.session_config_path(name, session_name).exists(): + # NOTE: Session wasn't launched from CLI + if system_config.is_session_configured(session_name): + raise errors.RenkulabSessionError( + "Session wasn't started using 'renku session start --ssh ...' " + "and is not configured for SSH access by you." + ) + communication.info(f"Setting up SSH connection config for session {session_name}") + system_config.setup_session_config(name, session_name) + ssh_setup = False + + exit_code = pty.spawn(["ssh", session_name]) + + if exit_code > 0 and not ssh_setup: + # NOTE: We tried to connect to SSH even though it wasn't started from CLI + # This failed, so we'll remove the temporary connection information. + if system_config.session_config_path(project_name, session_name).exists(): + system_config.session_config_path(project_name, session_name).unlink() + else: + url = self.session_url(session_name) + + webbrowser.open(url) + return True + def session_url(self, session_name: str) -> str: """Get the URL of the interactive session.""" project_name_parts = self._get_renku_project_name_parts() diff --git a/renku/core/session/session.py b/renku/core/session/session.py index 2ac81dbb79..97c2167cbc 100644 --- a/renku/core/session/session.py +++ b/renku/core/session/session.py @@ -17,8 +17,11 @@ # limitations under the License. """Interactive session business logic.""" -import webbrowser -from typing import List, Optional, Tuple +import os +import shutil +import textwrap +from pathlib import Path +from typing import List, NamedTuple, Optional from pydantic import validate_arguments @@ -28,19 +31,32 @@ from renku.core.session.utils import get_image_repository_host, get_renku_project_name from renku.core.util import communication from renku.core.util.os import safe_read_yaml +from renku.core.util.ssh import SystemSSHConfig, generate_ssh_keys from renku.domain_model.session import ISessionProvider, Session def _safe_get_provider(provider: str) -> ISessionProvider: try: - return next(p for p in get_supported_session_providers() if p.get_name() == provider) + return next(p for p in get_supported_session_providers() if p.name == provider) except StopIteration: raise errors.ParameterError(f"Session provider '{provider}' is not available!") +SessionList = NamedTuple( + "SessionList", [("sessions", List[Session]), ("all_local", bool), ("warning_messages", List[str])] +) + + @validate_arguments(config=dict(arbitrary_types_allowed=True)) -def session_list(config_path: Optional[str], provider: Optional[str] = None) -> Tuple[List[Session], bool, List[str]]: - """List interactive sessions.""" +def session_list(config_path: Optional[str], provider: Optional[str] = None) -> SessionList: + """List interactive sessions. + + Args: + config_path(str, optional): Path to config YAML. + provider(str, optional): Name of the session provider to use. + Returns: + The list of sessions, whether they're all local sessions and potential warnings raised. + """ def list_sessions(session_provider: ISessionProvider) -> List[Session]: try: @@ -62,26 +78,37 @@ def list_sessions(session_provider: ISessionProvider) -> List[Session]: try: sessions = list_sessions(session_provider) except errors.RenkuException as e: - warning_messages.append(f"Cannot get sessions list from '{session_provider.get_name()}': {e}") + warning_messages.append(f"Cannot get sessions list from '{session_provider.name}': {e}") else: if session_provider.is_remote_provider(): all_local = False all_sessions.extend(sessions) - return all_sessions, all_local, warning_messages + return SessionList(all_sessions, all_local, warning_messages) @validate_arguments(config=dict(arbitrary_types_allowed=True)) def session_start( - provider: str, config_path: Optional[str], + provider: str, image_name: Optional[str] = None, cpu_request: Optional[float] = None, mem_request: Optional[str] = None, disk_request: Optional[str] = None, gpu_request: Optional[str] = None, + **kwargs, ): - """Start interactive session.""" + """Start interactive session. + + Args: + config_path(str, optional): Path to config YAML. + provider(str, optional): Name of the session provider to use. + image_name(str, optional): Image to start. + cpu_request(float, optional): Number of CPUs to request. + mem_request(str, optional): Size of memory to request. + disk_request(str, optional): Size of disk to request (if supported by provider). + gpu_request(str, optional): Number of GPUs to request. + """ from renku.domain_model.project_context import project_context pinned_image = get_value("interactive", "image") @@ -91,7 +118,7 @@ def session_start( provider_api = _safe_get_provider(provider) config = safe_read_yaml(config_path) if config_path else dict() - provider_api.pre_start_checks() + provider_api.pre_start_checks(**kwargs) project_name = get_renku_project_name() if image_name is None: @@ -103,7 +130,8 @@ def session_start( if not provider_api.find_image(image_name, config): communication.confirm( - f"The container image '{image_name}' does not exists. Would you like to build it?", abort=True + f"The container image '{image_name}' does not exist. Would you like to build it using {provider}?", + abort=True, ) with communication.busy(msg=f"Building image {image_name}"): provider_api.build_image(project_context.docker_path.parent, image_name, config) @@ -134,6 +162,7 @@ def session_start( mem_request=mem_limit, disk_request=disk_limit, gpu_request=gpu, + **kwargs, ) if warning_message: @@ -143,7 +172,13 @@ def session_start( @validate_arguments(config=dict(arbitrary_types_allowed=True)) def session_stop(session_name: Optional[str], stop_all: bool = False, provider: Optional[str] = None): - """Stop interactive session.""" + """Stop interactive session. + + Args: + session_name(str): Name of the session to open. + stop_all(bool): Whether to stop all sessions or just the specified one. + provider(str, optional): Name of the session provider to use. + """ def stop_sessions(session_provider: ISessionProvider) -> bool: try: @@ -167,7 +202,7 @@ def stop_sessions(session_provider: ISessionProvider) -> bool: try: is_stopped = stop_sessions(session_provider) except errors.RenkuException as e: - warning_messages.append(f"Cannot stop sessions in provider '{session_provider.get_name()}': {e}") + warning_messages.append(f"Cannot stop sessions in provider '{session_provider.name}': {e}") if is_stopped and session_name: break @@ -183,21 +218,92 @@ def stop_sessions(session_provider: ISessionProvider) -> bool: @validate_arguments(config=dict(arbitrary_types_allowed=True)) -def session_open(session_name: str, provider: Optional[str] = None): - """Open interactive session in the browser.""" +def session_open(session_name: str, provider: Optional[str] = None, **kwargs): + """Open interactive session in the browser. - def open_sessions(session_provider: ISessionProvider) -> Optional[str]: - try: - return session_provider.session_url(session_name=session_name) - except errors.RenkulabSessionGetUrlError: - if provider: - raise - return None + Args: + session_name(str): Name of the session to open. + provider(str, optional): Name of the session provider to use. + """ providers = [_safe_get_provider(provider)] if provider else get_supported_session_providers() + project_name = get_renku_project_name() - url = next(filter(lambda u: u is not None, map(open_sessions, providers)), None) + found = False + for session_provider in providers: + if session_provider.session_open(project_name, session_name, **kwargs): + found = True + break - if url is None: + if not found: raise errors.ParameterError(f"Could not find '{session_name}' among the running sessions.") - webbrowser.open(url) + + +@validate_arguments(config=dict(arbitrary_types_allowed=True)) +def ssh_setup(existing_key: Optional[Path] = None, force: bool = False): + """Setup SSH keys for SSH connections to sessions. + + Args: + existing_key(Path, optional): Existing private key file to use instead of generating new ones. + force(bool): Whether to prompt before overwriting keys or not + """ + + if not shutil.which("ssh"): + raise errors.SSHNotFoundError() + + system_config = SystemSSHConfig() + + include_string = f"Include {system_config.renku_ssh_root}/*.conf\n\n" + + if include_string not in system_config.ssh_config.read_text(): + with system_config.ssh_config.open(mode="r+") as f: + content = f.read() + f.seek( + 0, 0 + ) # NOTE: We need to add 'Include' before any 'Host' entry, otherwise it is included as part of a host + f.write(include_string + content) + + if not existing_key and not force and system_config.is_configured: + communication.confirm(f"Keys already configured for host {system_config.renku_host}. Overwrite?", abort=True) + + if existing_key: + communication.info("Linking existing keys") + existing_public_key = existing_key.parent / (existing_key.name + ".pub") + + if not existing_key.exists() or not existing_public_key.exists(): + raise errors.KeyNotFoundError( + f"Couldn't find private key '{existing_key}' or public key '{existing_public_key}'." + ) + + if system_config.keyfile.exists(): + system_config.keyfile.unlink() + if system_config.public_keyfile.exists(): + system_config.public_keyfile.unlink() + + os.symlink(existing_key, system_config.keyfile) + os.symlink(existing_public_key, system_config.public_keyfile) + else: + communication.info("Generating keys") + keys = generate_ssh_keys() + system_config.keyfile.touch(mode=0o600) + system_config.public_keyfile.touch(mode=0o644) + with system_config.keyfile.open( + "wt", + ) as f: + f.write(keys.private_key) + + with system_config.public_keyfile.open("wt") as f: + f.write(keys.public_key) + + communication.info("Writing SSH config") + with system_config.jumphost_file.open(mode="wt") as f: + # NOTE: The * at the end of the jumphost name hides it from VSCode + content = textwrap.dedent( + f""" + Host jumphost-{system_config.renku_host}* + HostName {system_config.renku_host} + Port 2022 + User jovyan + """ + ) + f.write(content) diff --git a/renku/core/session/utils.py b/renku/core/session/utils.py index ae49e525cf..5b3758c225 100644 --- a/renku/core/session/utils.py +++ b/renku/core/session/utils.py @@ -16,6 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Utility functions used for sessions.""" +import os import urllib from typing import Optional @@ -48,6 +49,10 @@ def get_renku_url() -> Optional[str]: def get_image_repository_host() -> Optional[str]: """Derive the hostname for the gitlab container registry.""" + # NOTE: Used to circumvent cases where the registry URL is not guessed correctly + # remove once #3301 is done + if "RENKU_IMAGE_REGISTRY" in os.environ: + return os.environ["RENKU_IMAGE_REGISTRY"] renku_url = get_renku_url() if not renku_url: return None diff --git a/renku/core/storage.py b/renku/core/storage.py index a15699e7c5..24d88613f5 100644 --- a/renku/core/storage.py +++ b/renku/core/storage.py @@ -90,7 +90,7 @@ def check_external_storage_wrapper(fn): ``errors.ExternalStorageNotInstalled``: If external storage isn't installed. ``errors.ExternalStorageDisabled``: If external storage isn't enabled. """ - # noqa + @functools.wraps(fn) def wrapper(*args, **kwargs): if not check_external_storage(): @@ -219,11 +219,11 @@ def track_paths_in_storage(*paths): raise errors.ParameterError(f"Couldn't run 'git lfs':\n{e}") show_message = get_value("renku", "show_lfs_message") - if track_paths and (show_message is None or show_message == "True"): + if track_paths and (show_message is None or show_message.lower() == "true"): files_list = "\n\t".join(track_paths) communication.info( f"Adding these files to Git LFS:\n\t{files_list}" - "\nTo disable this message in the future, run:\n\trenku config set show_lfs_message False" + "\nTo disable this message in the future, run:\n\trenku config set show_lfs_message false" ) return track_paths diff --git a/renku/core/template/template.py b/renku/core/template/template.py index 927553061c..b7a3510c36 100644 --- a/renku/core/template/template.py +++ b/renku/core/template/template.py @@ -24,7 +24,7 @@ import tempfile from enum import Enum, IntEnum, auto from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, cast +from typing import Any, Dict, List, Optional, Tuple from packaging.version import Version @@ -132,6 +132,7 @@ def copy_template_metadata_to_project(): template_version=rendered_template.template.version, immutable_template_files=rendered_template.template.immutable_files.copy(), metadata=json.dumps(rendered_template.metadata), + ssh_supported=rendered_template.template.ssh_supported, ) actions_mapping: Dict[FileAction, Tuple[str, str]] = { @@ -482,10 +483,7 @@ def _has_template_at(self, id: str, reference: str) -> bool: """Return if template id is available at a reference.""" try: content = self.repository.get_content(TEMPLATE_MANIFEST, revision=reference) - - if isinstance(content, bytes): - return False - manifest = TemplatesManifest.from_string(cast(str, content)) + manifest = TemplatesManifest.from_string(content) except (errors.FileNotFound, errors.InvalidTemplateError): return False else: diff --git a/renku/core/util/git.py b/renku/core/util/git.py index 2a18a6da65..1dedf37018 100644 --- a/renku/core/util/git.py +++ b/renku/core/util/git.py @@ -196,7 +196,6 @@ def get_renku_repo_url(remote_url, deployment_hostname=None, access_token=None): remote_url: The repository URL. deployment_hostname: The host name used by this deployment (Default value = None). access_token: The OAuth2 access token (Default value = None). - Returns: The Renku repository URL with credentials. """ @@ -204,7 +203,8 @@ def get_renku_repo_url(remote_url, deployment_hostname=None, access_token=None): path = parsed_remote.path.strip("/") if path.startswith("gitlab/"): path = path.replace("gitlab/", "") - path = posixpath.join(CLI_GITLAB_ENDPOINT, path) + if not path.startswith(f"{CLI_GITLAB_ENDPOINT}/"): + path = posixpath.join(CLI_GITLAB_ENDPOINT, path) credentials = f"renku:{access_token}@" if access_token else "" hostname = deployment_hostname or parsed_remote.hostname diff --git a/renku/core/util/ssh.py b/renku/core/util/ssh.py new file mode 100644 index 0000000000..6a2008427f --- /dev/null +++ b/renku/core/util/ssh.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018-2022 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""SSH utility functions.""" + +import textwrap +import urllib.parse +from pathlib import Path +from typing import NamedTuple, Optional, cast + +from cryptography.hazmat.primitives import serialization as crypto_serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from renku.core import errors +from renku.core.session.utils import get_renku_url +from renku.core.util import communication +from renku.domain_model.project_context import project_context + +SSHKeyPair = NamedTuple("SSHKeyPair", [("private_key", str), ("public_key", str)]) + + +def generate_ssh_keys() -> SSHKeyPair: + """Generate an SSH keypair. + + Returns: + Private Public key pair. + """ + key = Ed25519PrivateKey.generate() + + private_key = key.private_bytes( + crypto_serialization.Encoding.PEM, + crypto_serialization.PrivateFormat.OpenSSH, + crypto_serialization.NoEncryption(), + ) + + public_key = key.public_key().public_bytes( + crypto_serialization.Encoding.OpenSSH, crypto_serialization.PublicFormat.OpenSSH + ) + + return SSHKeyPair(private_key.decode("utf-8"), public_key.decode("utf-8")) + + +class SystemSSHConfig: + """Class to manage system SSH config.""" + + def __init__(self) -> None: + """Initialize class and calculate paths.""" + self.ssh_root: Path = Path.home() / ".ssh" + self.ssh_config: Path = self.ssh_root / "config" + self.renku_ssh_root: Path = self.ssh_root / "renku" + + self.renku_ssh_root.mkdir(mode=0o700, exist_ok=True, parents=True) + self.ssh_config.touch(mode=0o600, exist_ok=True) + + self.renku_host: Optional[str] = cast(Optional[str], urllib.parse.urlparse(get_renku_url()).hostname) + + if not self.renku_host: + raise errors.AuthenticationError( + "Please use `renku login` to log in to the remote deployment before setting up ssh." + ) + + self.jumphost_file = self.renku_ssh_root / f"99-{self.renku_host}-jumphost.conf" + self.keyfile = self.renku_ssh_root / f"{self.renku_host}-key" + self.public_keyfile = self.renku_ssh_root / f"{self.renku_host}-key.pub" + + @property + def is_configured(self) -> bool: + """Check if the system is already configured correctly.""" + return self.jumphost_file.exists() and self.keyfile.exists() and self.public_keyfile.exists() + + @property + def public_key_string(self) -> Optional[str]: + """Get the public key string, ready for authorized_keys.""" + try: + key = self.public_keyfile.read_text() + key = f"\n{key} {project_context.repository.get_user().name}" + return key + except FileNotFoundError: + return None + + def is_session_configured(self, session_name: str) -> bool: + """Check if a session is configured for SSH. + + Args: + session_name(str): The name of the session. + """ + if not project_context.ssh_authorized_keys_path.exists(): + return False + + session_commit = session_name.rsplit("-", 1)[-1] + + try: + project_context.repository.get_commit(session_commit) + except errors.GitCommitNotFoundError: + return False + + try: + authorized_keys = project_context.repository.get_content( + project_context.ssh_authorized_keys_path, revision=session_commit + ) + except errors.FileNotFound: + return False + + if self.public_key_string and self.public_key_string in authorized_keys: + return True + return False + + def session_config_path(self, project_name: str, session_name: str) -> Path: + """Get path to a session config. + + Args: + project_name(str): The name of the project, without the owner name. + session_name(str): The name of the session to setup a connection to. + Returns: + The path to the SSH connection file. + """ + return self.renku_ssh_root / f"00-{project_name}-{session_name}.conf" + + def setup_session_keys(self): + """Add a users key to a project.""" + project_context.ssh_authorized_keys_path.parent.mkdir(parents=True, exist_ok=True) + project_context.ssh_authorized_keys_path.touch(mode=0o600, exist_ok=True) + + if not self.public_key_string: + raise errors.SSHNotSetupError() + + if self.public_key_string in project_context.ssh_authorized_keys_path.read_text(): + return + + communication.info("Adding SSH public key to project.") + with project_context.ssh_authorized_keys_path.open("at") as f: + f.writelines(self.public_key_string) + + project_context.repository.add(project_context.ssh_authorized_keys_path) + project_context.repository.commit("Add SSH public key.") + communication.info( + "Added public key. Changes need to be pushed and remote image built for changes to take effect." + ) + + def setup_session_config(self, project_name: str, session_name: str) -> str: + """Setup local SSH config for connecting to a session. + + Args: + project_name(str): The name of the project, without the owner name. + session_name(str): The name of the session to setup a connection to. + Returns: + The name of the created SSH host config. + """ + path = self.session_config_path(project_name, session_name) + path.touch(mode=0o600, exist_ok=True) + + config_content = textwrap.dedent( + f""" + Host {session_name} + HostName {session_name} + RemoteCommand cd work/{project_name}/ || true && exec $SHELL --login + RequestTTY yes + ServerAliveInterval 15 + ServerAliveCountMax 3 + ProxyJump jumphost-{self.renku_host} + IdentityFile {self.keyfile} + User jovyan + StrictHostKeyChecking no + """ + ) + + path.write_text(config_content) + + return session_name diff --git a/renku/domain_model/project.py b/renku/domain_model/project.py index 3d21bb4c52..d3d0ea3190 100644 --- a/renku/domain_model/project.py +++ b/renku/domain_model/project.py @@ -48,6 +48,7 @@ class ProjectTemplateMetadata: template_source: Optional[str] = None template_version: Optional[str] = None immutable_template_files: Optional[List[str]] = None + ssh_supported: bool = False class Project(persistent.Persistent): diff --git a/renku/domain_model/project_context.py b/renku/domain_model/project_context.py index 76c3a521c7..d64e2f806a 100644 --- a/renku/domain_model/project_context.py +++ b/renku/domain_model/project_context.py @@ -95,10 +95,15 @@ def dataset_images_path(self) -> Path: return self.path / RENKU_HOME / DATASET_IMAGES @property - def docker_path(self): + def docker_path(self) -> Path: """Path to the Dockerfile.""" return self.path / DOCKERFILE + @property + def ssh_authorized_keys_path(self) -> Path: + """Path to SSH authorized keys.""" + return self.path / ".ssh" / "authorized_keys" + @property def global_config_dir(self) -> str: """Return user's config directory.""" @@ -255,6 +260,7 @@ def push_path(self, path: Union[Path, str], save_changes: bool = False) -> None: Arguments: path(Union[Path, str]): The path to push. + save_changes(bool): Whether to save changes to the database or not. """ path = Path(path).resolve() self._context_stack.append(ProjectProperties(path=path, save_changes=save_changes)) @@ -280,6 +286,7 @@ def with_path( Arguments: path(Union[Path, str]): The path to push. + save_changes(bool): Whether to save changes to the database or not. """ with self.with_rollback(): self.push_path(path=path, save_changes=save_changes) diff --git a/renku/domain_model/session.py b/renku/domain_model/session.py index beaa7f14ef..51bfb11d9c 100644 --- a/renku/domain_model/session.py +++ b/renku/domain_model/session.py @@ -21,18 +21,22 @@ from abc import ABCMeta, abstractmethod from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from renku.core.constant import ProviderPriority +if TYPE_CHECKING: + from renku.core.dataset.providers.models import ProviderParameter + class Session: """Interactive session.""" - def __init__(self, id: str, status: str, url: str): + def __init__(self, id: str, status: str, url: str, ssh_enabled: bool = False): self.id = id self.status = status self.url = url + self.ssh_enabled = ssh_enabled class ISessionProvider(metaclass=ABCMeta): @@ -40,8 +44,9 @@ class ISessionProvider(metaclass=ABCMeta): priority: ProviderPriority = ProviderPriority.NORMAL + @property @abstractmethod - def get_name(self) -> str: + def name(self) -> str: """Return session provider's name.""" pass @@ -66,7 +71,7 @@ def find_image(self, image_name: str, config: Optional[Dict[str, Any]]) -> bool: """Search for the given container image. Args: - image_name: Container image name. + image_name(str): Container image name. config: Path to the session provider specific configuration YAML. Returns: @@ -83,13 +88,23 @@ def session_provider(self) -> "ISessionProvider": """ pass + @abstractmethod + def get_start_parameters(self) -> List["ProviderParameter"]: + """Returns parameters that can be set for session start.""" + pass + + @abstractmethod + def get_open_parameters(self) -> List["ProviderParameter"]: + """Returns parameters that can be set for session open.""" + pass + @abstractmethod def session_list(self, project_name: str, config: Optional[Dict[str, Any]]) -> List[Session]: """Lists all the sessions currently running by the given session provider. Args: - project_name: Renku project name. - config: Path to the session provider specific configuration YAML. + project_name(str): Renku project name. + config(Dict[str, Any], optional): Path to the session provider specific configuration YAML. Returns: a list of sessions. @@ -106,6 +121,7 @@ def session_start( mem_request: Optional[str] = None, disk_request: Optional[str] = None, gpu_request: Optional[str] = None, + **kwargs, ) -> Tuple[str, str]: """Creates an interactive session. @@ -128,9 +144,9 @@ def session_stop(self, project_name: str, session_name: Optional[str], stop_all: """Stops all or a given interactive session. Args: - project_name: Project's name. - session_name: The unique id of the interactive session. - stop_all: Specifies whether or not to stop all the running interactive sessions. + project_name(str): Project's name. + session_name(str, optional): The unique id of the interactive session. + stop_all(bool): Specifies whether or not to stop all the running interactive sessions. Returns: @@ -138,19 +154,29 @@ def session_stop(self, project_name: str, session_name: Optional[str], stop_all: """ pass + @abstractmethod + def session_open(self, project_name: str, session_name: str, **kwargs) -> bool: + """Open a given interactive session. + + Args: + project_name(str): Renku project name. + session_name(str): The unique id of the interactive session. + """ + pass + @abstractmethod def session_url(self, session_name: str) -> Optional[str]: """Get the given session's URL. Args: - session_name: The unique id of the interactive session. + session_name(str): The unique id of the interactive session. Returns: URL of the interactive session. """ pass - def pre_start_checks(self): + def pre_start_checks(self, **kwargs): """Perform any required checks on the state of the repository prior to starting a session. The expectation is that this method will abort the diff --git a/renku/domain_model/template.py b/renku/domain_model/template.py index 6d25ecaa31..84385afa2f 100644 --- a/renku/domain_model/template.py +++ b/renku/domain_model/template.py @@ -135,6 +135,7 @@ def templates(self) -> List["Template"]: description=cast(str, t.get("description")), parameters=cast(Dict[str, Dict[str, Any]], t.get("variables") or t.get("parameters")), icon=cast(str, t.get("icon")), + ssh_supported=t.get("ssh_supported", False), immutable_files=t.get("immutable_template_files", []), allow_update=t.get("allow_template_update", True), source=None, @@ -208,6 +209,7 @@ def __init__( description: str, parameters: Dict[str, Dict[str, Any]], icon: str, + ssh_supported: bool, immutable_files: List[str], allow_update: bool, source: Optional[str], @@ -224,6 +226,7 @@ def __init__( self.name: str = name self.description: str = description self.icon = icon + self.ssh_supported = ssh_supported self.immutable_files: List[str] = immutable_files or [] self.allow_update: bool = allow_update parameters = parameters or {} diff --git a/renku/infrastructure/repository.py b/renku/infrastructure/repository.py index 8de51970bc..5ad325ac17 100644 --- a/renku/infrastructure/repository.py +++ b/renku/infrastructure/repository.py @@ -35,6 +35,7 @@ Dict, Generator, List, + Literal, NamedTuple, Optional, Sequence, @@ -44,6 +45,7 @@ TypeVar, Union, cast, + overload, ) import git @@ -545,6 +547,28 @@ def get_ignored_paths(self, *paths: Union[Path, str]) -> List[str]: return ignored + @overload + def get_content( + self, + path: Union[Path, str], + *, + revision: Optional[Union["Reference", str]] = None, + checksum: Optional[str] = None, + binary: Literal[False] = False, + ) -> str: + ... + + @overload + def get_content( + self, + path: Union[Path, str], + *, + revision: Optional[Union["Reference", str]] = None, + checksum: Optional[str] = None, + binary: Literal[True], + ) -> bytes: + ... + def get_content( self, path: Union[Path, str], diff --git a/renku/ui/cli/config.py b/renku/ui/cli/config.py index ee2bbcb08d..ff7f36afb9 100644 --- a/renku/ui/cli/config.py +++ b/renku/ui/cli/config.py @@ -97,13 +97,19 @@ | ``lfs_threshold`` | Threshold file size below which | ``100kb`` | | | files are not added to git LFS | | +--------------------------------+-------------------------------------+-----------+ -| ``show_lfs_message`` | Whether to show messages about | ``True`` | +| ``show_lfs_message`` | Whether to show messages about | ``true`` | | | files being added to git LFS or not | | +--------------------------------+-------------------------------------+-----------+ -| ``show_login_warning`` | Whether to warn when logging in to | ``True`` | +| ``show_login_warning`` | Whether to warn when logging in to | ``true`` | | | Renku inside a project that causes | | | | project's git remote to be changed. | | +--------------------------------+-------------------------------------+-----------+ +| ``ssh_supported`` | Whether the image in this project | ``false`` | +| | contains an SSH server on port 22 | | +| | for SSH connections. Used to | | +| | override the corresponding project | | +| | template setting. | | ++--------------------------------+-------------------------------------+-----------+ | ``zenodo.access_token`` | Access token for Zenodo API | ``None`` | +--------------------------------+-------------------------------------+-----------+ diff --git a/renku/ui/cli/login.py b/renku/ui/cli/login.py index 270c231937..ad23b4b416 100644 --- a/renku/ui/cli/login.py +++ b/renku/ui/cli/login.py @@ -60,11 +60,11 @@ change the repository's remote URL to an endpoint in the deployment that adds authentication to gitlab requests. Renku warns you each time that the remote URL will change. To disable this warning, either pass ``--yes`` to the command -or set ``show_login_warning`` to ``False`` for the project or globally: +or set ``show_login_warning`` to ``false`` for the project or globally: .. code-block:: console - $ renku config set [--global] show_login_warning False + $ renku config set [--global] show_login_warning false You can avoid the remote from being changed by passing ``--no-git`` option to the login command. diff --git a/renku/ui/cli/project.py b/renku/ui/cli/project.py index 6c18428da8..d9f58349f3 100644 --- a/renku/ui/cli/project.py +++ b/renku/ui/cli/project.py @@ -150,6 +150,9 @@ def _print_project(project): click.echo( click.style("Project Template: ", bold=True, fg=color.MAGENTA) + click.style(project.template_info, bold=True) ) + click.echo( + click.style("SSH Supported: ", bold=True, fg=color.MAGENTA) + click.style(project.ssh_supported, bold=True) + ) if project.annotations: click.echo( diff --git a/renku/ui/cli/session.py b/renku/ui/cli/session.py index 10be3458af..2e5006ab13 100644 --- a/renku/ui/cli/session.py +++ b/renku/ui/cli/session.py @@ -92,6 +92,41 @@ $ renku session start -p renkulab +SSH connections to remote sessions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can connect via SSH to remote (Renkulab) sessions, if your project supports it. + +To see if your project supports SSH, you can run ``renku project show`` and check the +``SSH Supported`` flag. If your project doesn't support SSH, update the project template +or contact the template maintainer to enable SSH support on the template. + +You can start a session with SSH support using: + +.. code-block:: console + + $ renku session start -p renkulab --ssh + Your system is not set up for SSH connections to Renkulab. Would you like to set it up? [y/N]: y + [...] + Session sessionid successfully started, use 'renku session open --ssh sessionid' or 'ssh sessionid' to connect to it + +This will create SSH keys for you and setup SSH configuration for connecting to the renku deployment. +You can then use the SSH connection name (``ssh renkulab.io-myproject-sessionid`` in the example) +to connect to the session or in tools such as VSCode. + +.. note:: + + If you need to recreate the generated SSH keys or you want to use existing keys instead, + you can use the ``renku session ssh-setup`` command to perform this step manually. See + the help of the command for more details. + +Alternatively, you can use ``renku session open --ssh `` to directly open an SSH +connection to the session. + +You can see the SSH connection name using ``renku session ls``. + +SSH config for specific sessions is removed when the session is stopped. + Managing active sessions ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -188,18 +223,26 @@ def list_sessions(provider, config, format): from renku.command.session import session_list_command result = session_list_command().build().execute(provider=provider, config_path=config) - sessions, all_local, warning_messages = result.output - click.echo(SESSION_FORMATS[format](sessions)) + click.echo(SESSION_FORMATS[format](result.output.sessions)) - if warning_messages: + if result.output.warning_messages: click.echo() - if all_local and sessions: + if result.output.all_local and result.output.sessions: click.echo(WARNING + "Only showing sessions from local provider") - for message in warning_messages: + for message in result.output.warning_messages: click.echo(WARNING + message) +def session_start_provider_options(*param_decls, **attrs): + """Sets session provider options groups on the session start command.""" + from renku.core.plugin.session import get_supported_session_providers + from renku.ui.cli.utils.click import create_options + + providers = [p for p in get_supported_session_providers() if p.get_start_parameters()] + return create_options(providers=providers, parameter_function="get_start_parameters") + + @session.command("start") @click.option( "provider", @@ -223,7 +266,8 @@ def list_sessions(provider, config, format): @click.option("--disk", type=click.STRING, metavar="", help="Amount of disk space required for the session.") @click.option("--gpu", type=click.STRING, metavar="", help="GPU quota for the session.") @click.option("--memory", type=click.STRING, metavar="", help="Amount of memory required for the session.") -def start(provider, config, image, cpu, disk, gpu, memory): +@session_start_provider_options() +def start(provider, config, image, cpu, disk, gpu, memory, **kwargs): """Start an interactive session.""" from renku.command.session import session_start_command @@ -236,6 +280,7 @@ def start(provider, config, image, cpu, disk, gpu, memory): mem_request=memory, disk_request=disk, gpu_request=gpu, + **kwargs, ) @@ -269,6 +314,15 @@ def stop(session_name, stop_all, provider): click.echo(f"Interactive session '{session_name}' has been successfully stopped.") +def session_open_provider_options(*param_decls, **attrs): + """Sets session provider option groups on the session open command.""" + from renku.core.plugin.session import get_supported_session_providers + from renku.ui.cli.utils.click import create_options + + providers = [p for p in get_supported_session_providers() if p.get_open_parameters()] # type: ignore + return create_options(providers=providers, parameter_function="get_open_parameters") + + @session.command("open") @click.argument("session_name", metavar="", required=True) @click.option( @@ -279,8 +333,27 @@ def stop(session_name, stop_all, provider): default=None, help="Session provider to use.", ) -def open(session_name, provider): +@session_open_provider_options() +def open(session_name, provider, **kwargs): """Open an interactive session.""" from renku.command.session import session_open_command - session_open_command().build().execute(session_name=session_name, provider=provider) + session_open_command().build().execute(session_name=session_name, provider=provider, **kwargs) + + +@session.command("ssh-setup") +@click.option( + "existing_key", + "-k", + "--existing-key", + type=click.Path(exists=True, dir_okay=False), + metavar="", + help="Existing private key to use.", +) +@click.option("--force", is_flag=True, help="Overwrite existing keys/config.") +def ssh_setup(existing_key, force): + """Setup keys for SSH connections into sessions.""" + from renku.command.session import ssh_setup_command + + communicator = ClickCallback() + ssh_setup_command().with_communicator(communicator).build().execute(existing_key=existing_key, force=force) diff --git a/renku/ui/cli/utils/plugins.py b/renku/ui/cli/utils/plugins.py index 98444df508..55a09bf9e6 100644 --- a/renku/ui/cli/utils/plugins.py +++ b/renku/ui/cli/utils/plugins.py @@ -36,4 +36,4 @@ def get_supported_session_providers_names(): """Deferred import as plugins are slow.""" from renku.core.plugin.session import get_supported_session_providers - return [p.get_name() for p in get_supported_session_providers()] + return [p.name for p in get_supported_session_providers()] diff --git a/renku/ui/service/controllers/cache_migrations_check.py b/renku/ui/service/controllers/cache_migrations_check.py index 4bb7f64f2d..ff6317b935 100644 --- a/renku/ui/service/controllers/cache_migrations_check.py +++ b/renku/ui/service/controllers/cache_migrations_check.py @@ -58,7 +58,13 @@ def _fast_op_without_cache(self): tempdir_path = Path(tempdir) self.git_api_provider.download_files_from_api( - [".renku/metadata/root", ".renku/metadata/project", ".renku/metadata.yml", "Dockerfile"], + [ + ".renku/metadata/root", + ".renku/metadata/project", + ".renku/metadata.yml", + ".renku/renku.ini", + "Dockerfile", + ], tempdir_path, remote=self.ctx["git_url"], ref=self.request_data.get("ref", None), diff --git a/renku/ui/service/controllers/templates_create_project.py b/renku/ui/service/controllers/templates_create_project.py index 0b80f2e0f0..3febe94ef4 100644 --- a/renku/ui/service/controllers/templates_create_project.py +++ b/renku/ui/service/controllers/templates_create_project.py @@ -161,6 +161,7 @@ def new_project(self): commit_message=self.ctx["commit_message"], description=self.ctx["project_description"], data_dir=self.ctx.get("data_directory"), + ssh_supported=self.template.ssh_supported, ) self.new_project_push(new_project_path) diff --git a/renku/ui/service/serializers/cache.py b/renku/ui/service/serializers/cache.py index ed859cec26..a5861c0e38 100644 --- a/renku/ui/service/serializers/cache.py +++ b/renku/ui/service/serializers/cache.py @@ -339,6 +339,10 @@ class TemplateStatusResponse(Schema): allow_none=True, metadata={"description": "The current version of the template in the template repository."} ) + ssh_supported = fields.Boolean( + metadata={"description": "Whether this project supports ssh connections to sessions or not."} + ) + class ProjectMigrationCheckResponse(Schema): """Response schema for project migration check.""" diff --git a/renku/ui/service/serializers/templates.py b/renku/ui/service/serializers/templates.py index 92a2da803a..c4e3213cad 100644 --- a/renku/ui/service/serializers/templates.py +++ b/renku/ui/service/serializers/templates.py @@ -108,6 +108,9 @@ class ManifestTemplateSchema(Schema): icon = fields.String( load_default=None, metadata={"description": "base64 encoded icon for the template in PNG format"} ) + ssh_supported = fields.Boolean( + load_default=False, metadata={"description": "Whether the project supports SSH connections to sessions or not."} + ) class ManifestTemplatesResponse(Schema): diff --git a/renku/ui/service/serializers/v0_9/__init__.py b/renku/ui/service/serializers/v0_9/__init__.py deleted file mode 100644 index adbdcc123d..0000000000 --- a/renku/ui/service/serializers/v0_9/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2020 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Renku service v0.9 serializers.""" diff --git a/renku/ui/service/serializers/v0_9/cache.py b/renku/ui/service/serializers/v0_9/cache.py deleted file mode 100644 index eb69eb66d0..0000000000 --- a/renku/ui/service/serializers/v0_9/cache.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2020 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Renku service cache serializers.""" - -from marshmallow import Schema, fields - -from renku.ui.service.serializers.rpc import JsonRPCResponse - - -class ProjectMigrationCheckResponse_0_9(Schema): - """Response schema for project migration check.""" - - migration_required = fields.Boolean(attribute="core_compatibility_status.migration_required") - template_update_possible = fields.Boolean(attribute="template_status.newer_template_available") - automated_template_update = fields.Boolean(attribute="template_status.automated_template_update") - current_template_version = fields.String(attribute="template_status.project_template_version") - latest_template_version = fields.String(attribute="template_status.latest_template_version") - template_source = fields.String(attribute="template_status.template_source") - template_ref = fields.String(attribute="template_status.template_ref") - template_id = fields.String(attribute="template_status.template_id") - docker_update_possible = fields.Boolean(attribute="dockerfile_renku_status.newer_renku_available") - - project_supported = fields.Boolean() - project_renku_version = fields.String(data_key="project_version") - core_renku_version = fields.String(data_key="latest_version") - - -class ProjectMigrationCheckResponseRPC_0_9(JsonRPCResponse): - """RPC response schema for project migration check.""" - - result = fields.Nested(ProjectMigrationCheckResponse_0_9) diff --git a/renku/ui/service/serializers/v1_0/__init__.py b/renku/ui/service/serializers/v1/__init__.py similarity index 100% rename from renku/ui/service/serializers/v1_0/__init__.py rename to renku/ui/service/serializers/v1/__init__.py diff --git a/renku/ui/service/serializers/v1/cache.py b/renku/ui/service/serializers/v1/cache.py new file mode 100644 index 0000000000..96e140c19f --- /dev/null +++ b/renku/ui/service/serializers/v1/cache.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache serializers.""" + +from marshmallow import Schema, fields + +from renku.ui.service.serializers.cache import DockerfileStatusResponse, ProjectCompatibilityResponse +from renku.ui.service.serializers.common import RenkuSyncSchema +from renku.ui.service.serializers.rpc import JsonRPCResponse + + +class ProjectMigrateResponse_1_0(RenkuSyncSchema): + """Response schema for project migrate.""" + + was_migrated = fields.Boolean() + template_migrated = fields.Boolean() + docker_migrated = fields.Boolean() + messages = fields.List(fields.String) + + +class ProjectMigrateResponseRPC_1_0(JsonRPCResponse): + """RPC response schema for project migrate.""" + + result = fields.Nested(ProjectMigrateResponse_1_0) + + +class TemplateStatusResponse_1_5(Schema): + """Response schema outlining template status for migrations check.""" + + automated_template_update = fields.Boolean( + metadata={"description": "Whether or not the project template explicitly supports automated updates."} + ) + newer_template_available = fields.Boolean( + metadata={ + "description": "Whether or not the current version of the project template differs from the " + "one used in the project." + } + ) + template_source = fields.String( + metadata={ + "description": "Source of the template repository, " + "either a Git URL or 'renku' if an embedded template was used." + } + ) + template_ref = fields.String( + metadata={ + "description": "The branch/tag/commit from the template_source repository " + "that was used to create this project." + } + ) + template_id = fields.String(metadata={"description": "The id of the template in the template repository."}) + + project_template_version = fields.String( + allow_none=True, metadata={"description": "The version of the template last used in the user's project."} + ) + latest_template_version = fields.String( + allow_none=True, metadata={"description": "The current version of the template in the template repository."} + ) + + +class ProjectMigrationCheckResponse_1_5(Schema): + """Response schema for project migration check.""" + + project_supported = fields.Boolean( + metadata={ + "description": "Determines whether this project is a Renku project that is supported by the version " + "running on this service (not made with a newer version)." + } + ) + core_renku_version = fields.String(metadata={"description": "Version of Renku running in this service."}) + project_renku_version = fields.String(metadata={"description": "Version of Renku last used to change the project."}) + + core_compatibility_status = fields.Nested( + ProjectCompatibilityResponse, + metadata={"description": "Fields detailing the compatibility of the project with this core service version."}, + ) + dockerfile_renku_status = fields.Nested( + DockerfileStatusResponse, + metadata={"description": "Fields detailing the status of the Dockerfile in the project."}, + ) + template_status = fields.Nested( + TemplateStatusResponse_1_5, + metadata={"description": "Fields detailing the status of the project template used by this project."}, + ) + + +class ProjectMigrationCheckResponseRPC_1_5(JsonRPCResponse): + """RPC response schema for project migration check.""" + + result = fields.Nested(ProjectMigrationCheckResponse_1_5) diff --git a/renku/ui/service/serializers/v1/templates.py b/renku/ui/service/serializers/v1/templates.py new file mode 100644 index 0000000000..8fedce46da --- /dev/null +++ b/renku/ui/service/serializers/v1/templates.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service template serializers.""" + +from marshmallow import Schema, fields + +from renku.ui.service.serializers.rpc import JsonRPCResponse + + +class ManifestTemplateSchema_1_5(Schema): + """Manifest template schema.""" + + description = fields.String(required=True, metadata={"description": "Description of the template"}) + folder = fields.String(required=True, metadata={"description": "Folder the template resides in"}) + name = fields.String(required=True, metadata={"description": "Name of the template"}) + variables = fields.Dict( + load_default={}, metadata={"description": "Dictionary of values that can be set on this template"} + ) + icon = fields.String( + load_default=None, metadata={"description": "base64 encoded icon for the template in PNG format"} + ) + + +class ManifestTemplatesResponse_1_5(Schema): + """Manifest templates response.""" + + templates = fields.List(fields.Nested(ManifestTemplateSchema_1_5), required=True) + + +class ManifestTemplatesResponseRPC_1_5(JsonRPCResponse): + """RPC schema for listing manifest templates.""" + + result = fields.Nested(ManifestTemplatesResponse_1_5) diff --git a/renku/ui/service/serializers/v1_0/cache.py b/renku/ui/service/serializers/v1_0/cache.py deleted file mode 100644 index 0c417aac75..0000000000 --- a/renku/ui/service/serializers/v1_0/cache.py +++ /dev/null @@ -1,38 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2020 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Renku service cache serializers.""" - -from marshmallow import fields - -from renku.ui.service.serializers.common import RenkuSyncSchema -from renku.ui.service.serializers.rpc import JsonRPCResponse - - -class ProjectMigrateResponse_1_0(RenkuSyncSchema): - """Response schema for project migrate.""" - - was_migrated = fields.Boolean() - template_migrated = fields.Boolean() - docker_migrated = fields.Boolean() - messages = fields.List(fields.String) - - -class ProjectMigrateResponseRPC_1_0(JsonRPCResponse): - """RPC response schema for project migrate.""" - - result = fields.Nested(ProjectMigrateResponse_1_0) diff --git a/renku/ui/service/views/api_versions.py b/renku/ui/service/views/api_versions.py index 2959907fcb..6fb8bb2108 100644 --- a/renku/ui/service/views/api_versions.py +++ b/renku/ui/service/views/api_versions.py @@ -57,18 +57,18 @@ def add_url_rule( ) -V0_9 = ApiVersion("0.9") V1_0 = ApiVersion("1.0") V1_1 = ApiVersion("1.1") V1_2 = ApiVersion("1.2") V1_3 = ApiVersion("1.3") V1_4 = ApiVersion("1.4") -V1_5 = ApiVersion("1.5", is_base_version=True) +V1_5 = ApiVersion("1.5") +V2_0 = ApiVersion("2.0", is_base_version=True) -VERSIONS_FROM_V1_4 = [V1_4, V1_5] -VERSIONS_FROM_V1_1 = [V1_1, V1_2, V1_3, V1_4, V1_5] -VERSIONS_FROM_V1_0 = [V1_0] + VERSIONS_FROM_V1_1 -ALL_VERSIONS = [V0_9] + VERSIONS_FROM_V1_0 +VERSIONS_FROM_V1_5 = [V1_5, V2_0] +VERSIONS_FROM_V1_4 = [V1_4] + VERSIONS_FROM_V1_5 +VERSIONS_FROM_V1_1 = [V1_1, V1_2, V1_3] + VERSIONS_FROM_V1_4 +ALL_VERSIONS = [V1_0] + VERSIONS_FROM_V1_1 -MINIMUM_VERSION = V0_9 -MAXIMUM_VERSION = V1_5 +MINIMUM_VERSION = V1_0 +MAXIMUM_VERSION = V2_0 diff --git a/renku/ui/service/views/cache.py b/renku/ui/service/views/cache.py index 2a137b391c..f57a3f90e2 100644 --- a/renku/ui/service/views/cache.py +++ b/renku/ui/service/views/cache.py @@ -27,15 +27,14 @@ from renku.ui.service.controllers.cache_migrations_check import MigrationsCheckCtrl from renku.ui.service.controllers.cache_project_clone import ProjectCloneCtrl from renku.ui.service.gateways.gitlab_api_provider import GitlabAPIProvider -from renku.ui.service.views.api_versions import ALL_VERSIONS, VERSIONS_FROM_V1_0, VERSIONS_FROM_V1_1, VersionedBlueprint +from renku.ui.service.views.api_versions import ALL_VERSIONS, V2_0, VERSIONS_FROM_V1_1, VersionedBlueprint from renku.ui.service.views.decorators import accepts_json, optional_identity, requires_cache, requires_identity from renku.ui.service.views.error_handlers import ( handle_common_except, handle_migration_read_errors, handle_migration_write_errors, ) -from renku.ui.service.views.v0_9.cache import add_v0_9_specific_endpoints -from renku.ui.service.views.v1_0.cache import add_v1_0_specific_endpoints +from renku.ui.service.views.v1.cache import add_v1_specific_endpoints CACHE_BLUEPRINT_TAG = "cache" cache_blueprint = VersionedBlueprint("cache", __name__, url_prefix=SERVICE_PREFIX) @@ -209,9 +208,7 @@ def migrate_project_view(user_data, cache): return MigrateProjectCtrl(cache, user_data, dict(request.json)).to_response() # type: ignore -@cache_blueprint.route( - "/cache.migrations_check", methods=["GET"], provide_automatic_options=False, versions=VERSIONS_FROM_V1_0 -) +@cache_blueprint.route("/cache.migrations_check", methods=["GET"], provide_automatic_options=False, versions=[V2_0]) @handle_common_except @handle_migration_read_errors @requires_cache @@ -238,5 +235,4 @@ def migration_check_project_view(user_data, cache): return MigrationsCheckCtrl(cache, user_data, dict(request.args), GitlabAPIProvider()).to_response() -cache_blueprint = add_v0_9_specific_endpoints(cache_blueprint) -cache_blueprint = add_v1_0_specific_endpoints(cache_blueprint) +cache_blueprint = add_v1_specific_endpoints(cache_blueprint) diff --git a/renku/ui/service/views/graph.py b/renku/ui/service/views/graph.py index f81661680c..9d033e73d5 100644 --- a/renku/ui/service/views/graph.py +++ b/renku/ui/service/views/graph.py @@ -20,7 +20,7 @@ from renku.ui.service.config import SERVICE_PREFIX from renku.ui.service.controllers.graph_export import GraphExportCtrl -from renku.ui.service.views.api_versions import VERSIONS_FROM_V1_0, VersionedBlueprint +from renku.ui.service.views.api_versions import ALL_VERSIONS, VersionedBlueprint from renku.ui.service.views.decorators import accepts_json, optional_identity, requires_cache from renku.ui.service.views.error_handlers import handle_common_except, handle_graph_errors @@ -28,7 +28,7 @@ graph_blueprint = VersionedBlueprint(GRAPH_BLUEPRINT_TAG, __name__, url_prefix=SERVICE_PREFIX) -@graph_blueprint.route("/graph.export", methods=["GET"], provide_automatic_options=False, versions=VERSIONS_FROM_V1_0) +@graph_blueprint.route("/graph.export", methods=["GET"], provide_automatic_options=False, versions=ALL_VERSIONS) @handle_common_except @handle_graph_errors @requires_cache diff --git a/renku/ui/service/views/project.py b/renku/ui/service/views/project.py index 94f27c7a15..cd9e1a2724 100644 --- a/renku/ui/service/views/project.py +++ b/renku/ui/service/views/project.py @@ -22,7 +22,7 @@ from renku.ui.service.controllers.project_edit import ProjectEditCtrl from renku.ui.service.controllers.project_lock_status import ProjectLockStatusCtrl from renku.ui.service.controllers.project_show import ProjectShowCtrl -from renku.ui.service.views.api_versions import VERSIONS_FROM_V1_0, VersionedBlueprint +from renku.ui.service.views.api_versions import ALL_VERSIONS, VersionedBlueprint from renku.ui.service.views.decorators import accepts_json, requires_cache, requires_identity from renku.ui.service.views.error_handlers import handle_common_except, handle_project_write_errors @@ -30,9 +30,7 @@ project_blueprint = VersionedBlueprint(PROJECT_BLUEPRINT_TAG, __name__, url_prefix=SERVICE_PREFIX) -@project_blueprint.route( - "/project.show", methods=["POST"], provide_automatic_options=False, versions=VERSIONS_FROM_V1_0 -) +@project_blueprint.route("/project.show", methods=["POST"], provide_automatic_options=False, versions=ALL_VERSIONS) @handle_common_except @accepts_json @requires_cache @@ -60,9 +58,7 @@ def show_project_view(user_data, cache): return ProjectShowCtrl(cache, user_data, dict(request.json)).to_response() # type: ignore -@project_blueprint.route( - "/project.edit", methods=["POST"], provide_automatic_options=False, versions=VERSIONS_FROM_V1_0 -) +@project_blueprint.route("/project.edit", methods=["POST"], provide_automatic_options=False, versions=ALL_VERSIONS) @handle_common_except @handle_project_write_errors @accepts_json @@ -94,7 +90,7 @@ def edit_project_view(user_data, cache): @project_blueprint.route( - "/project.lock_status", methods=["GET"], provide_automatic_options=False, versions=VERSIONS_FROM_V1_0 + "/project.lock_status", methods=["GET"], provide_automatic_options=False, versions=ALL_VERSIONS ) @handle_common_except @requires_cache diff --git a/renku/ui/service/views/templates.py b/renku/ui/service/views/templates.py index 397cd2960f..1a01535eba 100644 --- a/renku/ui/service/views/templates.py +++ b/renku/ui/service/views/templates.py @@ -21,20 +21,21 @@ from renku.ui.service.config import SERVICE_PREFIX from renku.ui.service.controllers.templates_create_project import TemplatesCreateProjectCtrl from renku.ui.service.controllers.templates_read_manifest import TemplatesReadManifestCtrl -from renku.ui.service.views.api_versions import ALL_VERSIONS, VersionedBlueprint +from renku.ui.service.views.api_versions import ALL_VERSIONS, V2_0, VersionedBlueprint from renku.ui.service.views.decorators import accepts_json, requires_cache, requires_identity from renku.ui.service.views.error_handlers import ( handle_common_except, handle_templates_create_errors, handle_templates_read_errors, ) +from renku.ui.service.views.v1.templates import add_v1_specific_endpoints TEMPLATES_BLUEPRINT_TAG = "templates" templates_blueprint = VersionedBlueprint(TEMPLATES_BLUEPRINT_TAG, __name__, url_prefix=SERVICE_PREFIX) @templates_blueprint.route( - "/templates.read_manifest", methods=["GET"], provide_automatic_options=False, versions=ALL_VERSIONS + "/templates.read_manifest", methods=["GET"], provide_automatic_options=False, versions=[V2_0] ) @handle_common_except @handle_templates_read_errors @@ -102,3 +103,6 @@ def create_project_from_template(user_data, cache): - templates """ return TemplatesCreateProjectCtrl(cache, user_data, dict(request.json)).to_response() # type: ignore + + +templates_blueprint = add_v1_specific_endpoints(templates_blueprint) diff --git a/renku/ui/service/views/v0_9/__init__.py b/renku/ui/service/views/v0_9/__init__.py deleted file mode 100644 index 232f64e4e4..0000000000 --- a/renku/ui/service/views/v0_9/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2020 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Renku service v0.9 views.""" diff --git a/renku/ui/service/views/v0_9/cache.py b/renku/ui/service/views/v0_9/cache.py deleted file mode 100644 index 5dd183dedf..0000000000 --- a/renku/ui/service/views/v0_9/cache.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2020 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Renku service cache views.""" -from flask import request - -from renku.ui.service.controllers.cache_migrations_check import MigrationsCheckCtrl -from renku.ui.service.gateways.gitlab_api_provider import GitlabAPIProvider -from renku.ui.service.serializers.v0_9.cache import ProjectMigrationCheckResponseRPC_0_9 -from renku.ui.service.views.api_versions import V0_9 -from renku.ui.service.views.decorators import optional_identity, requires_cache -from renku.ui.service.views.error_handlers import handle_common_except - - -@handle_common_except -@requires_cache -@optional_identity -def migration_check_project_view_0_9(user_data, cache): - """ - Retrieve migration information for a project. - - --- - get: - description: Retrieve migration information for a project. - parameters: - - in: query - schema: ProjectMigrationCheckRequest - responses: - 200: - description: Information about required migrations for the project. - content: - application/json: - schema: ProjectMigrationCheckResponseRPC_0_9 - tags: - - cache - """ - ctrl = MigrationsCheckCtrl(cache, user_data, dict(request.args), GitlabAPIProvider()) - ctrl.RESPONSE_SERIALIZER = ProjectMigrationCheckResponseRPC_0_9() # type: ignore - return ctrl.to_response() - - -def add_v0_9_specific_endpoints(cache_blueprint): - """Add v0.9 only endpoints to blueprint.""" - cache_blueprint.route("/cache.migrations_check", methods=["GET"], provide_automatic_options=False, versions=[V0_9])( - migration_check_project_view_0_9 - ) - return cache_blueprint diff --git a/renku/ui/service/views/v1_0/__init__.py b/renku/ui/service/views/v1/__init__.py similarity index 100% rename from renku/ui/service/views/v1_0/__init__.py rename to renku/ui/service/views/v1/__init__.py diff --git a/renku/ui/service/views/v1/cache.py b/renku/ui/service/views/v1/cache.py new file mode 100644 index 0000000000..7beb4dae94 --- /dev/null +++ b/renku/ui/service/views/v1/cache.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service cache views.""" +from flask import request + +from renku.ui.service.controllers.cache_migrate_project import MigrateProjectCtrl +from renku.ui.service.controllers.cache_migrations_check import MigrationsCheckCtrl +from renku.ui.service.gateways.gitlab_api_provider import GitlabAPIProvider +from renku.ui.service.serializers.v1.cache import ProjectMigrateResponseRPC_1_0, ProjectMigrationCheckResponseRPC_1_5 +from renku.ui.service.views.api_versions import V1_0, V1_1, V1_2, V1_3, V1_4, V1_5 +from renku.ui.service.views.decorators import accepts_json, optional_identity, requires_cache, requires_identity +from renku.ui.service.views.error_handlers import ( + handle_common_except, + handle_migration_read_errors, + handle_migration_write_errors, +) + + +@handle_common_except +@handle_migration_write_errors +@accepts_json +@requires_cache +@requires_identity +def migrate_project_view_1_0(user_data, cache): + """ + Migrate a project. + + --- + post: + description: Migrate a project. + requestBody: + content: + application/json: + schema: ProjectMigrateRequest + responses: + 200: + description: Migration status. + content: + application/json: + schema: ProjectMigrateResponseRPC_1_0 + tags: + - cache + """ + ctrl = MigrateProjectCtrl(cache, user_data, dict(request.json)) # type: ignore + ctrl.RESPONSE_SERIALIZER = ProjectMigrateResponseRPC_1_0() # type: ignore + return ctrl.to_response() + + +@handle_common_except +@handle_migration_read_errors +@requires_cache +@optional_identity +def migration_check_project_view_1_5(user_data, cache): + """ + Retrieve migration information for a project. + + --- + get: + description: Retrieve migration information for a project. + parameters: + - in: query + schema: ProjectMigrationCheckRequest + responses: + 200: + description: Information about required migrations for the project. + content: + application/json: + schema: ProjectMigrationCheckResponseRPC_1_5 + tags: + - cache + """ + ctrl = MigrationsCheckCtrl(cache, user_data, dict(request.args), GitlabAPIProvider()) + ctrl.RESPONSE_SERIALIZER = ProjectMigrationCheckResponseRPC_1_5() # type: ignore + return ctrl.to_response() + + +def add_v1_specific_endpoints(cache_blueprint): + """Add v1 only endpoints to blueprint.""" + cache_blueprint.route("/cache.migrate", methods=["POST"], provide_automatic_options=False, versions=[V1_0])( + migrate_project_view_1_0 + ) + cache_blueprint.route( + "/cache.migrations_check", + methods=["GET"], + provide_automatic_options=False, + versions=[V1_0, V1_1, V1_2, V1_3, V1_4, V1_5], + )(migration_check_project_view_1_5) + return cache_blueprint diff --git a/renku/ui/service/views/v1/templates.py b/renku/ui/service/views/v1/templates.py new file mode 100644 index 0000000000..95d9534117 --- /dev/null +++ b/renku/ui/service/views/v1/templates.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 - Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku service templates view.""" +from flask import request + +from renku.ui.service.controllers.templates_read_manifest import TemplatesReadManifestCtrl +from renku.ui.service.serializers.v1.templates import ManifestTemplatesResponseRPC_1_5 +from renku.ui.service.views.api_versions import V1_0, V1_1, V1_2, V1_3, V1_4, V1_5 +from renku.ui.service.views.decorators import requires_cache, requires_identity +from renku.ui.service.views.error_handlers import handle_common_except, handle_templates_read_errors + + +@handle_common_except +@handle_templates_read_errors +@requires_cache +@requires_identity +def read_manifest_from_template_1_5(user_data, cache): + """ + Read templates from the manifest file of a template repository. + + --- + get: + description: Read templates from the manifest file of a template repository. + parameters: + - in: query + name: url + required: true + schema: + type: string + - in: query + name: ref + schema: + type: string + - in: query + name: depth + schema: + type: string + responses: + 200: + description: Listing of templates in the repository. + content: + application/json: + schema: ManifestTemplatesResponseRPC + tags: + - templates + """ + ctrl = TemplatesReadManifestCtrl(cache, user_data, dict(request.args)) + ctrl.RESPONSE_SERIALIZER = ManifestTemplatesResponseRPC_1_5() # type: ignore + + return ctrl.to_response() + + +def add_v1_specific_endpoints(templates_blueprint): + """Add v1 only endpoints to blueprint.""" + templates_blueprint.route( + "/templates.read_manifest", + methods=["GET"], + provide_automatic_options=False, + versions=[V1_0, V1_1, V1_2, V1_3, V1_4, V1_5], + )(read_manifest_from_template_1_5) + return templates_blueprint diff --git a/renku/ui/service/views/v1_0/cache.py b/renku/ui/service/views/v1_0/cache.py deleted file mode 100644 index 1fe4b00ce6..0000000000 --- a/renku/ui/service/views/v1_0/cache.py +++ /dev/null @@ -1,63 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2020 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Renku service cache views.""" -from flask import request - -from renku.ui.service.controllers.cache_migrate_project import MigrateProjectCtrl -from renku.ui.service.serializers.v1_0.cache import ProjectMigrateResponseRPC_1_0 -from renku.ui.service.views.api_versions import V0_9, V1_0 -from renku.ui.service.views.decorators import accepts_json, requires_cache, requires_identity -from renku.ui.service.views.error_handlers import handle_common_except, handle_migration_write_errors - - -@handle_common_except -@handle_migration_write_errors -@accepts_json -@requires_cache -@requires_identity -def migrate_project_view_1_0(user_data, cache): - """ - Migrate a project. - - --- - post: - description: Migrate a project. - requestBody: - content: - application/json: - schema: ProjectMigrateRequest - responses: - 200: - description: Migration status. - content: - application/json: - schema: ProjectMigrateResponseRPC_1_0 - tags: - - cache - """ - ctrl = MigrateProjectCtrl(cache, user_data, dict(request.json)) # type: ignore - ctrl.RESPONSE_SERIALIZER = ProjectMigrateResponseRPC_1_0() # type: ignore - return ctrl.to_response() - - -def add_v1_0_specific_endpoints(cache_blueprint): - """Add v1.0 only endpoints to blueprint.""" - cache_blueprint.route("/cache.migrate", methods=["POST"], provide_automatic_options=False, versions=[V0_9, V1_0])( - migrate_project_view_1_0 - ) - return cache_blueprint diff --git a/renku/ui/service/views/workflow_plans.py b/renku/ui/service/views/workflow_plans.py index 30d0c44eef..a729b6e3e1 100644 --- a/renku/ui/service/views/workflow_plans.py +++ b/renku/ui/service/views/workflow_plans.py @@ -22,7 +22,7 @@ from renku.ui.service.controllers.workflow_plans_export import WorkflowPlansExportCtrl from renku.ui.service.controllers.workflow_plans_list import WorkflowPlansListCtrl from renku.ui.service.controllers.workflow_plans_show import WorkflowPlansShowCtrl -from renku.ui.service.views.api_versions import V1_5, VERSIONS_FROM_V1_4, VersionedBlueprint +from renku.ui.service.views.api_versions import VERSIONS_FROM_V1_4, VERSIONS_FROM_V1_5, VersionedBlueprint from renku.ui.service.views.decorators import optional_identity, requires_cache from renku.ui.service.views.error_handlers import handle_common_except, handle_workflow_errors @@ -90,7 +90,7 @@ def show_plan_view(user_data, cache): @workflow_plans_blueprint.route( - "/workflow_plans.export", methods=["POST"], provide_automatic_options=False, versions=[V1_5] + "/workflow_plans.export", methods=["POST"], provide_automatic_options=False, versions=VERSIONS_FROM_V1_5 ) @handle_common_except @handle_workflow_errors diff --git a/tests/cli/fixtures/cli_projects.py b/tests/cli/fixtures/cli_projects.py index 7042b9bf1c..dc08d37047 100644 --- a/tests/cli/fixtures/cli_projects.py +++ b/tests/cli/fixtures/cli_projects.py @@ -66,7 +66,7 @@ def no_lfs_warning(project): For those times in life when mocking just isn't enough. """ - set_value("renku", "show_lfs_message", "False") + set_value("renku", "show_lfs_message", "false") project.repository.add(all=True) project.repository.commit(message="Unset show_lfs_message") @@ -81,7 +81,7 @@ def project_with_lfs_warning(project): with project_context.with_path(project.path): set_value("renku", "lfs_threshold", "0b") - set_value("renku", "show_lfs_message", "True") + set_value("renku", "show_lfs_message", "true") project.repository.add(".renku/renku.ini") project.repository.commit("update renku.ini") diff --git a/tests/cli/test_login.py b/tests/cli/test_login.py index 2fa72be713..33b9c5469e 100644 --- a/tests/cli/test_login.py +++ b/tests/cli/test_login.py @@ -48,7 +48,7 @@ def test_login(runner, project_with_remote, mock_login, with_injection): def test_login_with_no_warn_config(runner, project_with_remote, mock_login, with_injection): """Test login command with ``show_login_warning`` configured in the ``renku.ini``.""" - assert 0 == runner.invoke(cli, ["config", "set", "show_login_warning", "False"]).exit_code + assert 0 == runner.invoke(cli, ["config", "set", "show_login_warning", "false"]).exit_code result = runner.invoke(cli, ["login", ENDPOINT]) # No ``--yes`` and no input @@ -158,7 +158,7 @@ def test_login_to_multiple_endpoints(runner, project_with_remote, mock_login, wi with with_injection(): assert ACCESS_TOKEN == read_renku_token(ENDPOINT) assert second_token == read_renku_token(second_endpoint) - assert project_with_remote.repository.remotes["origin"].url.startswith(f"https://{ENDPOINT}/repo") + assert project_with_remote.repository.remotes["origin"].url.startswith(f"https://{second_endpoint}/repo") def test_logout_all(runner, project, mock_login, with_injection): diff --git a/tests/cli/test_migrate.py b/tests/cli/test_migrate.py index 2a44a3d10b..e37264f6e0 100644 --- a/tests/cli/test_migrate.py +++ b/tests/cli/test_migrate.py @@ -89,6 +89,7 @@ def test_migration_check(isolated_runner, project): "latest_template_version", "template_ref", "template_id", + "ssh_supported", } diff --git a/tests/cli/test_session.py b/tests/cli/test_session.py index b0d978601b..b864f87748 100644 --- a/tests/cli/test_session.py +++ b/tests/cli/test_session.py @@ -18,7 +18,7 @@ """Test ``service`` command.""" import re -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from renku.ui.cli import cli from tests.utils import format_result_exception @@ -26,7 +26,7 @@ def test_session_up_down(runner, project, dummy_session_provider, monkeypatch): """Test starting a session.""" - import renku.core.session.session + browser = dummy_session_provider result = runner.invoke(cli, ["session", "ls", "-p", "dummy"]) assert 0 == result.exit_code, format_result_exception(result) @@ -43,12 +43,9 @@ def test_session_up_down(runner, project, dummy_session_provider, monkeypatch): assert 0 == result.exit_code, format_result_exception(result) assert 5 == len(result.output.splitlines()) - with monkeypatch.context() as monkey: - browser = MagicMock() - monkey.setattr(renku.core.session.session, "webbrowser", browser) - result = runner.invoke(cli, ["session", "open", "-p", "dummy", session_id]) - assert 0 == result.exit_code, format_result_exception(result) - browser.open.assert_called_once_with("http://localhost/") + result = runner.invoke(cli, ["session", "open", "-p", "dummy", session_id]) + assert 0 == result.exit_code, format_result_exception(result) + browser.open.assert_called_once_with("http://localhost/") result = runner.invoke(cli, ["session", "stop", "-p", "dummy", session_id]) assert 0 == result.exit_code, format_result_exception(result) @@ -84,3 +81,27 @@ def test_session_start_config_requests(runner, project, dummy_session_provider, result = runner.invoke(cli, ["session", "start", "-p", "docker"]) assert 0 == result.exit_code, format_result_exception(result) assert "successfully started" in result.output + + +def test_session_ssh_setup(runner, project, dummy_session_provider, fake_home): + """Test starting a session.""" + from renku.core.util.ssh import generate_ssh_keys + + with patch("renku.core.util.ssh.get_renku_url", lambda: "https://renkulab.io/"): + result = runner.invoke(cli, ["session", "ssh-setup"]) + assert 0 == result.exit_code, format_result_exception(result) + assert "Generating keys" in result.output + assert "Writing SSH config" in result.output + + private_key, public_key = generate_ssh_keys() + + private_path = fake_home / ".ssh" / "existing" + private_path.write_text(private_key) + (fake_home / ".ssh" / "existing.pub").write_text(public_key) + + with patch("renku.core.util.ssh.get_renku_url", lambda: "https://renkulab.io/"): + result = runner.invoke(cli, ["session", "ssh-setup", "-k", str(private_path), "--force"]) + assert 0 == result.exit_code, format_result_exception(result) + + assert "Linking existing keys" in result.output + assert "Writing SSH config" in result.output diff --git a/tests/core/plugins/test_session.py b/tests/core/plugins/test_session.py index 47f795250c..e54f937a7a 100644 --- a/tests/core/plugins/test_session.py +++ b/tests/core/plugins/test_session.py @@ -17,15 +17,18 @@ # limitations under the License. """Test ``session`` commands.""" +import re from unittest.mock import patch +import click import pytest -from renku.core.errors import ParameterError +from renku.core.errors import ParameterError, RenkulabSessionError from renku.core.plugin.session import get_supported_session_providers from renku.core.session.docker import DockerSessionProvider from renku.core.session.renkulab import RenkulabSessionProvider -from renku.core.session.session import session_list, session_start, session_stop +from renku.core.session.session import session_list, session_start, session_stop, ssh_setup +from renku.core.util.ssh import SystemSSHConfig def fake_start( @@ -37,6 +40,7 @@ def fake_start( mem_request, disk_request, gpu_request, + **kwargs, ): return "0xdeadbeef", "" @@ -61,7 +65,7 @@ def fake_session_list(self, project_name, config): return ["0xdeadbeef"] -def fake_pre_start_checks(self): +def fake_pre_start_checks(self, **kwargs): pass @@ -91,6 +95,7 @@ def test_session_start( with_injection, mock_communication, ): + """Test starting sessions.""" with patch.multiple( session_provider, session_start=fake_start, @@ -100,7 +105,7 @@ def test_session_start( **provider_patches, ): provider_implementation = next( - filter(lambda x: x.get_name() == provider_name, get_supported_session_providers()), None + filter(lambda x: x.name == provider_name, get_supported_session_providers()), None ) assert provider_implementation is not None @@ -138,9 +143,10 @@ def test_session_stop( result, with_injection, ): + """Test stopping sessions.""" with patch.multiple(session_provider, session_stop=fake_stop, **provider_patches): provider_implementation = next( - filter(lambda x: x.get_name() == provider_name, get_supported_session_providers()), None + filter(lambda x: x.name == provider_name, get_supported_session_providers()), None ) assert provider_implementation is not None @@ -161,7 +167,6 @@ def test_session_stop( ) @pytest.mark.parametrize("provider_exists,result", [(True, ["0xdeadbeef"]), (False, ParameterError)]) def test_session_list( - run_shell, project, provider_name, session_provider, @@ -170,6 +175,7 @@ def test_session_list( result, with_injection, ): + """Test listing sessions.""" with patch.multiple(session_provider, session_list=fake_session_list, **provider_patches): with with_injection(): provider = provider_name if provider_exists else "no_provider" @@ -178,5 +184,123 @@ def test_session_list( with pytest.raises(result): session_list(provider=provider, config_path=None) else: - sessions, _, _ = session_list(provider=provider, config_path=None) - assert sessions == result + output = session_list(provider=provider, config_path=None) + assert output.sessions == result + + +def test_session_ssh_setup(project, with_injection, fake_home, mock_communication): + """Test setting up SSH config for a deployment.""" + with patch("renku.core.util.ssh.get_renku_url", lambda: "https://renkulab.io/"): + with with_injection(): + ssh_setup() + + ssh_home = fake_home / ".ssh" + renku_ssh_path = ssh_home / "renku" + assert renku_ssh_path.exists() + assert re.search(r"Include .*/\.ssh/renku/\*\.conf", (ssh_home / "config").read_text()) + assert (renku_ssh_path / "99-renkulab.io-jumphost.conf").exists() + assert (renku_ssh_path / "renkulab.io-key").exists() + assert (renku_ssh_path / "renkulab.io-key.pub").exists() + assert len(mock_communication.confirm_calls) == 0 + + key = (renku_ssh_path / "renkulab.io-key").read_text() + + with patch("renku.core.util.ssh.get_renku_url", lambda: "https://renkulab.io/"): + with with_injection(): + with pytest.raises(click.Abort): + ssh_setup() + + assert len(mock_communication.confirm_calls) == 1 + assert key == (renku_ssh_path / "renkulab.io-key").read_text() + + with patch("renku.core.util.ssh.get_renku_url", lambda: "https://renkulab.io/"): + with with_injection(): + ssh_setup(force=True) + + assert key != (renku_ssh_path / "renkulab.io-key").read_text() + + +def test_session_start_ssh(project, with_injection, mock_communication, fake_home): + """Test starting of a session with SSH support.""" + from renku.domain_model.project_context import project_context + + def _fake_send_request(self, req_type: str, *args, **kwargs): + class _FakeResponse: + status_code = 200 + + def json(self): + return {"name": "0xdeadbeef"} + + return _FakeResponse() + + with patch.multiple( + RenkulabSessionProvider, + find_image=fake_find_image, + build_image=fake_build_image, + _wait_for_session_status=lambda _, __, ___: None, + _send_renku_request=_fake_send_request, + _remote_head_hexsha=lambda _: project.repository.head.commit.hexsha, + _renku_url=lambda _: "example.com", + _cleanup_ssh_connection_configs=lambda _, __: None, + _auth_header=lambda _: None, + ): + provider_implementation = next(filter(lambda x: x.name == "renkulab", get_supported_session_providers()), None) + assert provider_implementation is not None + + with patch("renku.core.util.ssh.get_renku_url", lambda: "https://renkulab.io/"): + with with_injection(): + ssh_setup() + supported = project_context.project.template_metadata.ssh_supported + + try: + project_context.project.template_metadata.ssh_supported = False + with pytest.raises(expected_exception=RenkulabSessionError, match="project doesn't support SSH"): + session_start(provider="renkulab", config_path=None, ssh=True) + finally: + project_context.project.template_metadata.ssh_supported = supported + + with patch("renku.core.util.ssh.get_renku_url", lambda: "https://renkulab.io/"): + with with_injection(): + supported = project_context.project.template_metadata.ssh_supported + + try: + project_context.project.template_metadata.ssh_supported = True + session_start(provider="renkulab", config_path=None, ssh=True) + finally: + project_context.project.template_metadata.ssh_supported = supported + + assert any("0xdeadbeef" in line for line in mock_communication.stdout_lines) + ssh_home = fake_home / ".ssh" + renku_ssh_path = ssh_home / "renku" + assert (renku_ssh_path / "99-renkulab.io-jumphost.conf").exists() + assert (project.path / ".ssh" / "authorized_keys").exists() + assert len(list(renku_ssh_path.glob("00-*-0xdeadbeef.conf"))) == 1 + + +def test_session_ssh_configured(project, with_injection, fake_home): + """Test that the SSH class correctly determines if a session is configured for SSH.""" + from renku.domain_model.project_context import project_context + + with patch("renku.core.util.ssh.get_renku_url", lambda: "https://renkulab.io/"): + with with_injection(): + system_config = SystemSSHConfig() + + assert not system_config.is_session_configured("my-session-abcdefg") + + previous_commit = project_context.repository.head.commit + + project_context.ssh_authorized_keys_path.parent.mkdir(parents=True, exist_ok=True) + project_context.ssh_authorized_keys_path.touch() + project_context.repository.add(project_context.ssh_authorized_keys_path) + intermediate_commit = project_context.repository.commit("Add auth keys file") + + assert not system_config.is_session_configured("my-session-abcdefg") + assert not system_config.is_session_configured(f"my-session-{previous_commit}") + assert not system_config.is_session_configured(f"my-session-{intermediate_commit}") + + key = "my-key" + system_config.public_keyfile.write_text(key) + project_context.ssh_authorized_keys_path.write_text(f"\n{key} Renku Bot") + project_context.repository.add(project_context.ssh_authorized_keys_path) + valid_commit = project_context.repository.commit("Add auth keys file") + assert system_config.is_session_configured(f"my-session-{valid_commit}") diff --git a/tests/fixtures/communication.py b/tests/fixtures/communication.py index 7a7d6fb842..89f3ab3b56 100644 --- a/tests/fixtures/communication.py +++ b/tests/fixtures/communication.py @@ -19,6 +19,7 @@ from typing import Generator, List +import click import pytest from renku.core.util.communication import CommunicationCallback @@ -31,6 +32,7 @@ def __init__(self): super().__init__() self._stdout: List[str] = [] self._stderr: List[str] = [] + self.confirm_calls: List[str] = [] @property def stdout(self) -> str: @@ -70,8 +72,15 @@ def error(self, msg): def confirm(self, msg, abort=False, warning=False, default=False): """Get confirmation for an action.""" + self.confirm_calls.append(msg) + if abort: + raise click.Abort() return False + def has_prompt(self): + """Prompt for user input.""" + return True + def _write_to(self, message: str, output=None): output = output or self._stdout with CommunicationCallback.lock: diff --git a/tests/fixtures/repository.py b/tests/fixtures/repository.py index 670d67216e..aa47020b58 100644 --- a/tests/fixtures/repository.py +++ b/tests/fixtures/repository.py @@ -69,7 +69,7 @@ def isolated_filesystem(path: Path, name: str = None, delete: bool = True): if delete: try: shutil.rmtree(base_path) - except OSError: # noqa: B014 + except OSError: pass @@ -89,7 +89,7 @@ def fake_home(tmp_path, monkeypatch) -> Generator[Path, None, None]: global_config.set_value("user", "email", "renku@datascience.ch") global_config.set_value("pull", "rebase", "false") - set_value(section="renku", key="show_lfs_message", value="False", global_only=True) + set_value(section="renku", key="show_lfs_message", value="false", global_only=True) yield home diff --git a/tests/fixtures/session.py b/tests/fixtures/session.py index ebea8d3b75..3d00a194e6 100644 --- a/tests/fixtures/session.py +++ b/tests/fixtures/session.py @@ -18,6 +18,7 @@ """Renku session fixtures.""" from typing import Any, Dict, List, Optional, Tuple +from unittest.mock import MagicMock import pytest @@ -35,7 +36,8 @@ def dummy_session_provider(): class _DummySessionProvider(ISessionProvider): sessions = list() - def get_name(self): + @property + def name(self): return "dummy" def is_remote_provider(self): @@ -63,6 +65,7 @@ def session_start( mem_request: Optional[str] = None, disk_request: Optional[str] = None, gpu_request: Optional[str] = None, + **kwargs, ) -> Tuple[str, str]: name = f"session-random-{uuid4().hex}-name" self.sessions.append(name) @@ -79,13 +82,25 @@ def session_stop(self, project_name: str, session_name: Optional[str], stop_all: def session_url(self, session_name: str) -> Optional[str]: return "http://localhost/" - def pre_start_checks(self): + def pre_start_checks(self, **kwargs): pass + def get_start_parameters(self): + return [] + + def get_open_parameters(self): + return [] + + def session_open(self, project_name: str, session_name: str, **kwargs) -> bool: + browser.open(self.session_url(session_name)) + return True + plugin = _DummySessionProvider() pm = plugin_manager.get_plugin_manager() pm.register(plugin) - yield + browser = MagicMock() + + yield browser pm.unregister(plugin) diff --git a/tests/fixtures/templates.py b/tests/fixtures/templates.py index f8c2d01342..f4164535c6 100644 --- a/tests/fixtures/templates.py +++ b/tests/fixtures/templates.py @@ -137,6 +137,7 @@ def source_template(tmp_path): path=dummy_template_root, icon="", templates_source=None, + ssh_supported=False, ) diff --git a/tests/service/views/test_api_versions_views.py b/tests/service/views/test_api_versions_views.py index bc2d27d518..c8e8e951ba 100644 --- a/tests/service/views/test_api_versions_views.py +++ b/tests/service/views/test_api_versions_views.py @@ -35,19 +35,29 @@ def test_versions_differences(svc_client, identity_headers, it_remote_repo_url): ) assert 200 == response_old.status_code - assert response_old.json["result"]["migration_required"] is True + assert response_old.json["result"]["core_compatibility_status"]["migration_required"] is True + assert "ssh_supported" not in response_old.json["result"]["template_status"] response_new = svc_client.get( f"/{max_ver}/cache.migrations_check", query_string=query_string, headers=identity_headers ) assert 200 == response_new.status_code assert response_new.json["result"]["core_compatibility_status"]["migration_required"] is True + assert "ssh_supported" in response_new.json["result"]["template_status"] response_default = svc_client.get("/cache.migrations_check", query_string=query_string, headers=identity_headers) assert 200 == response_default.status_code assert response_default.json["result"]["core_compatibility_status"]["migration_required"] is True assert response_default.json["result"].keys() == response_new.json["result"].keys() - assert response_default.json["result"].keys() != response_old.json["result"].keys() + assert response_default.json["result"].keys() == response_old.json["result"].keys() + assert ( + response_default.json["result"]["template_status"].keys() + == response_new.json["result"]["template_status"].keys() + ) + assert ( + response_default.json["result"]["template_status"].keys() + != response_old.json["result"]["template_status"].keys() + ) assert ( response_default.json["result"]["core_compatibility_status"]["migration_required"] == response_new.json["result"]["core_compatibility_status"]["migration_required"] diff --git a/tests/service/views/test_cache_views.py b/tests/service/views/test_cache_views.py index 93ab72c58e..fbfffa8916 100644 --- a/tests/service/views/test_cache_views.py +++ b/tests/service/views/test_cache_views.py @@ -877,6 +877,8 @@ def test_check_migrations_local(svc_client_setup): assert "template_ref" in response.json["result"]["template_status"] assert "template_id" in response.json["result"]["template_status"] assert "automated_template_update" in response.json["result"]["template_status"] + assert "ssh_supported" in response.json["result"]["template_status"] + assert not response.json["result"]["template_status"]["ssh_supported"] @pytest.mark.service diff --git a/tests/service/views/test_version_views.py b/tests/service/views/test_version_views.py index c176ddb189..f3e346d34e 100644 --- a/tests/service/views/test_version_views.py +++ b/tests/service/views/test_version_views.py @@ -36,7 +36,7 @@ def test_version(svc_client): assert MINIMUM_VERSION.name == data["minimum_api_version"] assert MAXIMUM_VERSION.name == data["maximum_api_version"] - response = svc_client.get("/0.9/apiversion") + response = svc_client.get("/1.0/apiversion") assert "result" in response.json data = response.json["result"] @@ -48,7 +48,7 @@ def test_version(svc_client): assert MINIMUM_VERSION.name == data["minimum_api_version"] assert MAXIMUM_VERSION.name == data["maximum_api_version"] - response = svc_client.get("/1.0/apiversion") + response = svc_client.get("/1.1/apiversion") assert "result" in response.json data = response.json["result"] @@ -60,7 +60,7 @@ def test_version(svc_client): assert MINIMUM_VERSION.name == data["minimum_api_version"] assert MAXIMUM_VERSION.name == data["maximum_api_version"] - response = svc_client.get("/1.1/apiversion") + response = svc_client.get("/1.2/apiversion") assert "result" in response.json data = response.json["result"] @@ -72,7 +72,7 @@ def test_version(svc_client): assert MINIMUM_VERSION.name == data["minimum_api_version"] assert MAXIMUM_VERSION.name == data["maximum_api_version"] - response = svc_client.get("/1.2/apiversion") + response = svc_client.get("/2.0/apiversion") assert "result" in response.json data = response.json["result"] diff --git a/tests/service/views/v0_9/test_cache_views_0_9.py b/tests/service/views/v0_9/test_cache_views_0_9.py deleted file mode 100644 index d5ea61fb76..0000000000 --- a/tests/service/views/v0_9/test_cache_views_0_9.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2019-2022 - Swiss Data Science Center (SDSC) -# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Renku service cache view tests.""" - -import pytest - - -@pytest.mark.service -@pytest.mark.integration -def test_check_migrations_local_0_9(svc_client_setup): - """Check if migrations are required for a local project.""" - svc_client, headers, project_id, _, _ = svc_client_setup - - response = svc_client.get("/0.9/cache.migrations_check", query_string=dict(project_id=project_id), headers=headers) - assert 200 == response.status_code - - assert response.json["result"]["migration_required"] - assert not response.json["result"]["template_update_possible"] - assert not response.json["result"]["docker_update_possible"] - assert response.json["result"]["project_supported"] - assert response.json["result"]["project_version"] - assert response.json["result"]["latest_version"] - assert "template_source" in response.json["result"] - assert "template_ref" in response.json["result"] - assert "template_id" in response.json["result"] - assert "automated_template_update" in response.json["result"] - - -@pytest.mark.service -@pytest.mark.integration -def test_check_migrations_remote_0_9(svc_client, identity_headers, it_remote_repo_url): - """Check if migrations are required for a remote project.""" - response = svc_client.get( - "/0.9/cache.migrations_check", query_string=dict(git_url=it_remote_repo_url), headers=identity_headers - ) - - assert 200 == response.status_code - - assert response.json["result"]["migration_required"] - assert not response.json["result"]["template_update_possible"] - assert not response.json["result"]["docker_update_possible"] - assert response.json["result"]["project_supported"] - assert response.json["result"]["project_version"] - assert response.json["result"]["latest_version"] - - -@pytest.mark.service -@pytest.mark.integration -def test_check_no_migrations_0_9(svc_client_with_repo): - """Check if migrations are not required.""" - svc_client, headers, project_id, _ = svc_client_with_repo - - response = svc_client.get("/0.9/cache.migrations_check", query_string=dict(project_id=project_id), headers=headers) - - assert 200 == response.status_code - assert not response.json["result"]["migration_required"] - assert not response.json["result"]["template_update_possible"] - assert not response.json["result"]["docker_update_possible"] - assert response.json["result"]["project_supported"] - - -@pytest.mark.service -@pytest.mark.integration -def test_check_migrations_remote_anonymous_0_9(svc_client, it_remote_public_repo_url): - """Test anonymous users can check for migration of public projects.""" - response = svc_client.get( - "/0.9/cache.migrations_check", query_string={"git_url": it_remote_public_repo_url}, headers={} - ) - - assert 200 == response.status_code - - assert response.json["result"]["migration_required"] is True From 9eb4f9a5e80b91ed2daefb42a3b9b430dddbc51e Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Mon, 27 Feb 2023 15:28:50 +0100 Subject: [PATCH 10/13] tests: fix code for tests with python 3.7 (#3335) --- renku/infrastructure/repository.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/renku/infrastructure/repository.py b/renku/infrastructure/repository.py index 5ad325ac17..29b25c9a90 100644 --- a/renku/infrastructure/repository.py +++ b/renku/infrastructure/repository.py @@ -35,7 +35,6 @@ Dict, Generator, List, - Literal, NamedTuple, Optional, Sequence, @@ -53,6 +52,11 @@ from renku.core import errors from renku.core.util.os import delete_dataset_file, get_absolute_path +try: + from typing_extensions import Literal # NOTE: Required for Python 3.7 compatibility +except ImportError: + from typing import Literal # type: ignore + NULL_TREE = git.NULL_TREE _MARKER = object() GIT_IGNORE = ".gitignore" From 62f69ab2244c0771dd633dc6837b5c506a04984d Mon Sep 17 00:00:00 2001 From: Renku Bot Date: Mon, 27 Feb 2023 15:15:37 +0000 Subject: [PATCH 11/13] chore: release v2.3.0 --- CHANGES.rst | 24 ++++++++++++++++++++++++ helm-chart/renku-core/Chart.yaml | 2 +- helm-chart/renku-core/values.yaml | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index cee2773e86..819124f6b6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -18,6 +18,30 @@ Changes ======= +`2.3.0 `__ (2023-02-27) +------------------------------------------------------------------------------------------------------- + +Bug Fixes +~~~~~~~~~ + +- **cli:** fix renku workflow visualize crashing for large graphs due + to a curses error + (`#3273 `__) + (`5898cbb `__) +- **core:** metadata v10 migration fixes + (`#3326 `__) + (`38aa53c `__) +- **svc:** properly use Redis sentinel + (`#3319 `__) + (`4a52069 `__) + +Features +~~~~~~~~ + +- **core:** add support for ssh connections into sessions + (`#3318 `__) + (`c024650 `__) + `2.2.0 `__ (2023-02-10) ------------------------------------------------------------------------------------------------------- diff --git a/helm-chart/renku-core/Chart.yaml b/helm-chart/renku-core/Chart.yaml index 3ef7daa171..8d51bc8a05 100644 --- a/helm-chart/renku-core/Chart.yaml +++ b/helm-chart/renku-core/Chart.yaml @@ -3,4 +3,4 @@ appVersion: "1.0" description: A Helm chart for Kubernetes name: renku-core icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4 -version: 2.2.0 +version: 2.3.0 diff --git a/helm-chart/renku-core/values.yaml b/helm-chart/renku-core/values.yaml index 584ed0b3de..e63f5451dd 100644 --- a/helm-chart/renku-core/values.yaml +++ b/helm-chart/renku-core/values.yaml @@ -97,7 +97,7 @@ versions: fullnameOverride: "" image: repository: renku/renku-core - tag: "v2.2.0" + tag: "v2.3.0" pullPolicy: IfNotPresent v9: name: v9 From 5a70f7869ebb846363719d7a094c0c0822f14b86 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Mon, 27 Feb 2023 17:35:01 +0100 Subject: [PATCH 12/13] fix PR action --- .github/workflows/test_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_deploy.yml b/.github/workflows/test_deploy.yml index 3d002c8127..477d2cad48 100644 --- a/.github/workflows/test_deploy.yml +++ b/.github/workflows/test_deploy.yml @@ -5,6 +5,7 @@ on: branches: - master - develop + - "release/*" # otherwise PRs created by the PR action don't execute tests tags: - "v*.*.*" pull_request: From 4d0538a3e5452b4276cdc1e3ec8444df399ff5d2 Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Mon, 27 Feb 2023 17:43:19 +0100 Subject: [PATCH 13/13] fix pull request action on release PRs --- .github/workflows/test_deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_deploy.yml b/.github/workflows/test_deploy.yml index 477d2cad48..af949f71b4 100644 --- a/.github/workflows/test_deploy.yml +++ b/.github/workflows/test_deploy.yml @@ -5,11 +5,13 @@ on: branches: - master - develop - - "release/*" # otherwise PRs created by the PR action don't execute tests + - "release/**" # otherwise PRs created by the PR action don't execute tests tags: - "v*.*.*" pull_request: types: [opened, reopened, synchronize] + branches: + - "!release/**" env: DEPENDENCY_CACHE_PREFIX: "v2"