From e986e218d82f951be518f6096da291749c718ee1 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 16 Sep 2024 10:18:30 +0200 Subject: [PATCH] feat: add OpenAPI configuration files and new SDK artifact upload workflow --- .github/openapi/python-config.json | 4 +- .../openapi/python-pydantic-v1-config.json | 5 + .github/openapi/typescript-config.json | 2 +- .github/workflows/github-tag-and-release.yml | 78 ------ .github/workflows/publish-sdk-packages.yml | 166 ------------ .github/workflows/release-and-publish-sdk.yml | 242 ++++++++++++++++++ .github/workflows/upload-sdk-artifact.yml | 82 ++++++ 7 files changed, 332 insertions(+), 247 deletions(-) create mode 100644 .github/openapi/python-pydantic-v1-config.json delete mode 100644 .github/workflows/github-tag-and-release.yml delete mode 100644 .github/workflows/publish-sdk-packages.yml create mode 100644 .github/workflows/release-and-publish-sdk.yml create mode 100644 .github/workflows/upload-sdk-artifact.yml diff --git a/.github/openapi/python-config.json b/.github/openapi/python-config.json index 1323b5f58..709747cf2 100644 --- a/.github/openapi/python-config.json +++ b/.github/openapi/python-config.json @@ -1,5 +1,5 @@ { "generatorName": "python", - "packageName": "Scicat-Python-SDK", - "projectName": "Scicat-Python-SDK" + "packageName": "scicat-sdk-py", + "projectName": "scicat-sdk-py" } diff --git a/.github/openapi/python-pydantic-v1-config.json b/.github/openapi/python-pydantic-v1-config.json new file mode 100644 index 000000000..ef10d66f2 --- /dev/null +++ b/.github/openapi/python-pydantic-v1-config.json @@ -0,0 +1,5 @@ +{ + "generatorName": "python-pydantic-v1", + "packageName": "scicat-sdk-pydantic", + "projectName": "scicat-sdk-pydantic" +} diff --git a/.github/openapi/typescript-config.json b/.github/openapi/typescript-config.json index 4248aecec..fc279bbcf 100644 --- a/.github/openapi/typescript-config.json +++ b/.github/openapi/typescript-config.json @@ -1,6 +1,6 @@ { "generatorName": "typescript-angular", - "npmName": "@scicat-sdk/typescript-angular", + "npmName": "@scicatproject/scicat-sdk-ts", "ngVersion": "16.2.12", "withInterfaces": true } diff --git a/.github/workflows/github-tag-and-release.yml b/.github/workflows/github-tag-and-release.yml deleted file mode 100644 index 6b3ddb5cc..000000000 --- a/.github/workflows/github-tag-and-release.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Bump release version and build-push - -on: - push: - branches: - - release - -env: - NODE_VERSION: 20.x - RELEASE_BRANCH: release - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - ## Commit message examples for Release type (patch|minor|major) can be found: - ## https://github.com/mathieudutour/github-tag-action - - name: Bump version and push tag - id: tag_version - uses: mathieudutour/github-tag-action@v6.2 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - release_branches: ${{ env.RELEASE_BRANCH }} - - - name: Create a GitHub release - uses: ncipollo/release-action@v1 - with: - tag: ${{ steps.tag_version.outputs.new_tag }} - name: Release ${{ steps.tag_version.outputs.new_tag }} - body: ${{ steps.tag_version.outputs.changelog }} - - ## The setup-qemu-action simplifies the setup of QEMU for cross-platform builds - ## https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - - - name: Install Node.js dependencies - run: npm ci - - ## The metadata-action dynamically generates and manages metadata for Docker images, - ## like tags and labels, based on the provided inputs and workflow context. - ## https://github.com/docker/metadata-action - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/scicatproject/backend-next - tags: | - type=raw,value=stable - type=raw,value=${{ steps.tag_version.outputs.new_tag }} - type=semver,pattern={{version}} - type=raw,value={{date 'YYYY_MM'}},prefix=r_ - - - name: Build and push - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64,linux/arm64/v8 - push: true - tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/publish-sdk-packages.yml b/.github/workflows/publish-sdk-packages.yml deleted file mode 100644 index fc0bc77ad..000000000 --- a/.github/workflows/publish-sdk-packages.yml +++ /dev/null @@ -1,166 +0,0 @@ -name: Generate and Publish SDK for multiple languages - -on: - workflow_run: - workflows: ["Bump release version and build-push"] - types: - - completed - -env: - NODE_VERSION: 18.x - PYTHON_VERSION: 3.x - -jobs: - start-backend-and-upload-swagger-schema: - runs-on: ubuntu-latest - outputs: - current-version: ${{ steps.package-version.outputs.current-version }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{env.NODE_VERSION}} - - - name: get-npm-version - id: package-version - uses: martinbeentjes/npm-get-version-action@v1.3.1 - - - name: Pull MongoDB Image - run: | - docker pull mongo:latest - docker run -d --name mongo-container -p 27017:27017 mongo:latest - - - name: Install Backend and wait for it to be ready - env: - MONGODB_URI: "mongodb://localhost:27017/scicat" - JWT_SECRET: thisIsTheJwtSecret - run: | - npm install -g wait-on && npm install - npm run start & wait-on http://localhost:3000/api/v3/health --timeout 200000 - - - name: Download the Swagger schema - run: curl -o ./swagger-schema.json http://localhost:3000/explorer-json - - - uses: actions/upload-artifact@v4 - with: - name: swagger-schema - path: ./swagger-schema.json - - generate-and-upload-sdk: - runs-on: ubuntu-latest - needs: start-backend-and-upload-swagger-schema - strategy: - matrix: - generator: [python, typescript] - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set packageVersion variable - run: echo "packageVersion=${{ needs.start-backend-and-upload-swagger-schema.outputs.current-version }}" >> $GITHUB_ENV - - - uses: actions/download-artifact@v4 - with: - name: swagger-schema - path: . - - - name: Check packageVersion - run: | - if [ -z "$packageVersion" ]; then - echo "packageVersion is not set, exiting..." - exit 1 - fi - - - name: Generate Client - uses: openapi-generators/openapitools-generator-action@v1 - with: - generator: ${{ matrix.generator }} - openapi-file: ./swagger-schema.json - config-file: .github/openapi/${{ matrix.generator }}-config.json - command-args: | - -o ./sdk/${{ matrix.generator }} $( - if [ "${{ matrix.generator }}" == "typescript" ]; then - echo "--additional-properties=npmVersion=$packageVersion"; - elif [ "${{ matrix.generator }}" == "python" ]; then - echo "--additional-properties=packageVersion=$packageVersion"; - fi - ) - - - uses: actions/upload-artifact@v4 - with: - name: sdk-${{ matrix.generator }} - path: ./sdk - - npm-publish: - needs: generate-and-upload-sdk - runs-on: ubuntu-latest - environment: - name: npm-sdk-package - url: https://www.npmjs.com/package/@scicat-sdk/typescript-angular - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - registry-url: "https://registry.npmjs.org/" - - - uses: actions/download-artifact@v4 - with: - name: sdk-typescript - path: ./sdk - - - name: Publish package - run: | - npm install - npm run build - npm publish --access public - working-directory: ./sdk/typescript/ - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - pypi-publish: - needs: generate-and-upload-sdk - runs-on: ubuntu-latest - environment: - name: pypi-sdk-package - url: https://pypi.org/p/Scicat-Python-SDK - permissions: - id-token: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - uses: actions/download-artifact@v4 - with: - name: sdk-python - path: ./sdk - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel - working-directory: ./sdk/python/ - - - name: Build package - run: | - python setup.py sdist bdist_wheel - working-directory: ./sdk/python/ - - - name: Publish package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: ./sdk/python/dist/ diff --git a/.github/workflows/release-and-publish-sdk.yml b/.github/workflows/release-and-publish-sdk.yml new file mode 100644 index 000000000..4228642e6 --- /dev/null +++ b/.github/workflows/release-and-publish-sdk.yml @@ -0,0 +1,242 @@ +name: Bump release version, build-push image and publish SDK + +on: + push: + branches: + - release + +env: + NODE_VERSION: 20.x + PYTHON_VERSION: 3.x + RELEASE_BRANCH: release + +jobs: + build-release: + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + + outputs: + new_tag: ${{ steps.without_v.outputs.tag }} + changelog: ${{ steps.tag_version.outputs.changelog }} + + steps: + - uses: actions/checkout@v4 + + ## Commit message examples for Release type (patch|minor|major) can be found: + ## https://github.com/mathieudutour/github-tag-action + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v6.2 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + release_branches: ${{ env.RELEASE_BRANCH }} + + - name: Strip 'v' from the tag + id: without_v + run: | + TAG=${{ steps.tag_version.outputs.new_tag }} + WITHOUT_V=${TAG#v} + echo "tag=$WITHOUT_V" >> $GITHUB_OUTPUT + + - name: Create a GitHub release + uses: ncipollo/release-action@v1 + with: + tag: ${{ steps.tag_version.outputs.new_tag }} + name: Release ${{ steps.tag_version.outputs.new_tag }} + body: ${{ steps.tag_version.outputs.changelog }} + + ## The setup-qemu-action simplifies the setup of QEMU for cross-platform builds + ## https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install Node.js dependencies + run: npm ci + + ## The metadata-action dynamically generates and manages metadata for Docker images, + ## like tags and labels, based on the provided inputs and workflow context. + ## https://github.com/docker/metadata-action + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/scicatproject/backend-next + tags: | + type=raw,value=stable + type=raw,value=${{ steps.tag_version.outputs.new_tag }} + type=semver,pattern={{version}} + type=raw,value={{date 'YYYY_MM'}},prefix=r_ + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64/v8 + push: true + tags: ${{ steps.meta.outputs.tags }} + + start-backend-export-swagger: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{env.NODE_VERSION}} + + - name: Pull and Run MongoDB + run: | + docker pull mongo:latest + docker run -d --name mongo-container -p 27017:27017 mongo:latest + + - name: Install Backend and wait for it to be ready + env: + MONGODB_URI: "mongodb://localhost:27017/scicat" + JWT_SECRET: thisIsTheJwtSecret + run: | + npm install -g wait-on && npm install + npm run start & wait-on http://localhost:3000/api/v3/health --timeout 200000 + + - name: Download the Swagger schema + run: curl -o ./swagger-schema.json http://localhost:3000/explorer-json + + - uses: actions/upload-artifact@v4 + with: + name: swagger-schema + path: ./swagger-schema.json + + generate-upload-sdk: + runs-on: ubuntu-latest + needs: + - build-release + - start-backend-export-swagger + strategy: + matrix: + generator: [python, python-pydantic-v1, typescript] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + name: swagger-schema + path: . + + - name: Generate Client + uses: openapi-generators/openapitools-generator-action@v1 + with: + generator: ${{ matrix.generator }} + openapi-file: ./swagger-schema.json + config-file: .github/openapi/${{ matrix.generator }}-config.json + command-args: | + --git-repo-id scicat-backend-next \ + --git-user-id SciCatProject \ + -o ./sdk/${{ matrix.generator }} $( + if [ "${{ matrix.generator }}" == "typescript" ]; then + echo "--additional-properties=npmVersion=${{ needs.build-release.outputs.new_tag}}"; + elif [ "${{ matrix.generator }}" == "python" ]; then + echo "--additional-properties=packageVersion=${{ needs.build-release.outputs.new_tag}}"; + elif [ "${{ matrix.generator }}" == "python-pydantic-v1" ]; then + echo "--additional-properties=packageVersion=${{ needs.build-release.outputs.new_tag}}"; + fi + ) + + - uses: actions/upload-artifact@v4 + with: + name: sdk-${{ matrix.generator }}-${{ github.sha }} + path: ./sdk + + npm-publish: + needs: generate-upload-sdk + runs-on: ubuntu-latest + environment: + name: npm-sdk-package + url: https://www.npmjs.com/package/@scicatproject/scicat-sdk-ts + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: "https://registry.npmjs.org/" + + - name: Download TypeScript SDK Artifact + uses: actions/download-artifact@v4 + with: + name: sdk-typescript-${{github.sha}} + path: ./sdk + + - name: Publish package + run: | + npm install + npm run build + npm publish --access public + working-directory: ./sdk/typescript/ + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + pypi-publish: + needs: generate-upload-sdk + runs-on: ubuntu-latest + strategy: + matrix: + sdk_type: [python, python-pydantic-v1] + environment: + name: ${{ matrix.sdk_type }}-sdk-package + url: ${{ matrix.sdk_type == 'python' && 'https://pypi.org/project/scicat-sdk-py' || 'https://pypi.org/project/scicat-sdk-pydantic' }} + permissions: + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Download Python SDK Artifact + uses: actions/download-artifact@v4 + with: + name: sdk-${{ matrix.sdk_type }}-${{github.sha}} + path: ./sdk + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel + working-directory: ./sdk/${{ matrix.sdk_type }}/ + + - name: Build package + run: | + python setup.py sdist bdist_wheel + working-directory: ./sdk/${{ matrix.sdk_type }}/ + + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: ./sdk/${{ matrix.sdk_type }}/dist/ diff --git a/.github/workflows/upload-sdk-artifact.yml b/.github/workflows/upload-sdk-artifact.yml new file mode 100644 index 000000000..cbce35fc4 --- /dev/null +++ b/.github/workflows/upload-sdk-artifact.yml @@ -0,0 +1,82 @@ +name: Generate and upload latest SDK artifacts + +on: + push: + branches: + - master + +env: + NODE_VERSION: 20.x + SDK_VERSION: latest + +jobs: + start-backend-and-upload-swagger-schema: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{env.NODE_VERSION}} + + - name: Pull MongoDB Image + run: | + docker pull mongo:latest + docker run -d --name mongo-container -p 27017:27017 mongo:latest + + - name: Install Backend and wait for it to be ready + env: + MONGODB_URI: "mongodb://localhost:27017/scicat" + JWT_SECRET: thisIsTheJwtSecret + run: | + npm install -g wait-on && npm install + npm run start & wait-on http://localhost:3000/api/v3/health --timeout 200000 + + - name: Download the Swagger schema + run: curl -o ./swagger-schema.json http://localhost:3000/explorer-json + + - uses: actions/upload-artifact@v4 + with: + name: swagger-schema + path: ./swagger-schema.json + + generate-and-upload-sdk: + runs-on: ubuntu-latest + needs: + - start-backend-and-upload-swagger-schema + strategy: + matrix: + generator: [python, typescript, python-pydantic-v1] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + name: swagger-schema + path: . + + - name: Generate Client + uses: openapi-generators/openapitools-generator-action@v1 + with: + generator: ${{ matrix.generator }} + openapi-file: ./swagger-schema.json + config-file: .github/openapi/${{ matrix.generator }}-config.json + command-args: | + -o ./sdk/${{ matrix.generator }} $( + if [ "${{ matrix.generator }}" == "typescript" ]; then + echo "--additional-properties=npmVersion=${{env.SDK_VERSION}}"; + elif [ "${{ matrix.generator }}" == "python" ]; then + echo "--additional-properties=packageVersion=${{env.SDK_VERSION}}"; + elif [ "${{ matrix.generator }}" == "python-pydantic-v1" ]; then + echo "--additional-properties=packageVersion=${{env.SDK_VERSION}}"; + fi + ) + + - uses: actions/upload-artifact@v4 + with: + name: sdk-${{ matrix.generator }}-${{ github.sha }} + path: ./sdk