Save OpenAPI definition as json, generate client and run tests

.github/openapi/javascript-config.yml

@@ -0,0 +1,5 @@
+packageName: "scicat-openapi-client"
+packageUrl: "javascript.package.url"
+projectName: "scicat-openapi-client"
+disallowAdditionalPropertiesIfNotPresent: false
+usePromises: true

.github/openapi/python-config.yml

@@ -0,0 +1,2 @@
+packageName: "scicat_openapi_client"
+packageUrl: "python.package.url"

.github/openapi/typescript-config.yml

@@ -0,0 +1,2 @@
+packageName: "scicat-openapi-client"
+packageUrl: "typescript.package.url"

.github/workflows/publish_openapi_client.yaml

@@ -0,0 +1,102 @@
+name: Generate & Publish OpenAPI Client
+env:
+  NODE_VERSION: 18.x
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  generate-openapi-definition:
+    name: Generate the OpenAPI definition and cache it
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Install dependencies
+        if: steps.cached-node-modules.outputs.cache-hit != 'true'
+        run: npm ci
+
+      - name: Build
+        run: npm run build
+
+      - name: Cache OpenAPI definition
+        id: cached-openapi-definition
+        uses: actions/cache@v4
+        with:
+          path: ./
+          key: openapi.json
+
+      - name: Generate OpenAPI definition
+        env:
+          EXIT_AFTER_OPENAPI_GENERATION: true
+          ELASTICSEARCH_ENABLED: false
+          MONGODB_URI: mongodb://localhost:27017/scicat
+          EXPRESS_SESSION_SECRET: a_scicat_secret
+          JWT_SECRET: a_scicat_secret
+          PORT: 3000
+          HTTP_MAX_REDIRECTS: 5
+          HTTP_TIMEOUT: 5000
+          JWT_EXPIRES_IN: 3600
+          LOGGING_DISABLED: true
+          SITE: PSI
+          PID_PREFIX: PID.SAMPLE.PREFIX
+          DOI_PREFIX: DOI.SAMPLE.PREFIX
+          METADATA_KEYS_RETURN_LIMIT: 100
+          METADATA_PARENT_INSTANCES_RETURN_LIMIT: 100
+          ADMIN_GROUPS: ingestor
+          DELETE_GROUPS: ingestor
+          CREATE_DATASET_GROUPS: ingestor
+          CREATE_DATASET_WITH_PID_GROUPS: ingestor
+          CREATE_DATASET_PRIVILEGED_GROUPS: ingestor
+          ACCESS_GROUPS_STATIC_VALUES: "ess"
+          PROPOSAL_GROUPS: ingestor
+          SAMPLE_GROUPS: ""
+          DATASET_CREATION_VALIDATION_ENABLED: false
+          MONGODB_COLLECTION: Dataset
+          MEM_LIMIT: 4G
+
+        # Start mongo container and app before starting
+        run: |
+          cp CI/ESS/docker-compose.api.yaml docker-compose.yaml
+          cp functionalAccounts.json.test functionalAccounts.json
+          docker-compose up --build -d
+          npm run start
+
+  generate-and-push:
+    name: Run install and cache
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        generator: ${{ toJSON(env.REPOSITORY_VAR) }}
+    steps:
+      - name: Restore OpenAPI definition
+        id: cached-openapi-definition
+        uses: actions/cache@v4
+        with:
+          path: ./
+          key: openapi.json
+
+      # Use the action to generate a client package
+      # This uses the default path for the openapi document and thus assumes there is an openapi.json in the current workspace.
+      # https://github.com/openapi-generators/openapitools-generator-action
+      # https://openapi-generator.tech/docs/generators/
+      - name: Generate Client
+        uses: openapi-generators/openapitools-generator-action@v1
+        with:
+          generator: ${{ matrix.generator }}
+          openapi-file: openapi.json
+          config-file: .github/openapi/${{ matrix.generator }}-config.yml
+
+      # Checkout client and force upload changes
+      - name: Git Push to its client library repository
+        run: |
+          cd ${{ matrix.generator }}
+          /bin/sh ./git_push.sh

.github/workflows/test.yml

@@ -45,6 +45,7 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
+
       - name: Cache node_modules
         id: cached-node-modules
         uses: actions/cache@v4
@@ -103,7 +104,7 @@ jobs:
 
   api_tests:
     name: API tests with ElasticSearch enabled - ${{ matrix.elasticsearch_enabled }}
-    needs: [install-and-cache]
+    needs: [ install-and-cache ]
     runs-on: ubuntu-latest
 
     strategy:
@@ -171,7 +172,162 @@ jobs:
 
         # Start mongo container and app before running api tests
         run: |
-          cp CI/ESS/docker-compose.api.yaml docker-compose.yaml
           cp functionalAccounts.json.test functionalAccounts.json
-          docker-compose up --build -d
+          docker-compose -f CI/ESS/docker-compose.api.yaml up --build -d
+          until curl -s -f -o /dev/null "localhost:27017"; do sleep 5; done
           npm run test:api
+
+  generate_openapi:
+    name: Generate OpenAPI definition and generate clients
+    needs: [ install-and-cache ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Restore node_modules
+        id: cached-node-modules
+        uses: actions/cache@v4
+        with:
+          path: node_modules
+          key: node-modules-${{ hashFiles('package-lock.json') }}-${{ env.NODE_VERSION }}
+
+      - name: Start backend
+        env:
+          EXIT_AFTER_OPENAPI_GENERATION: true
+          ELASTICSEARCH_ENABLED: no
+          MONGODB_URI: mongodb://localhost:27017/scicat
+          EXPRESS_SESSION_SECRET: a_scicat_secret
+          JWT_SECRET: a_scicat_secret
+          PORT: 3000
+          HTTP_MAX_REDIRECTS: 5
+          HTTP_TIMEOUT: 5000
+          JWT_EXPIRES_IN: 3600
+          LOGGING_DISABLED: True
+          # NOTE: The SITE variable is set to PSI because of the PublishedData test.
+          # Testing /register endpoint expects to have it as PSI for that specific scenario.
+          # Old backend was modifying this at runtime but with NestJS we need to investigate little bit more if there is a bit better and more elegant solution.
+          SITE: PSI
+          PID_PREFIX: PID.SAMPLE.PREFIX
+          DOI_PREFIX: DOI.SAMPLE.PREFIX
+          METADATA_KEYS_RETURN_LIMIT: 100
+          METADATA_PARENT_INSTANCES_RETURN_LIMIT: 100
+          ADMIN_GROUPS: admin,ingestor
+          DELETE_GROUPS: archivemanager
+          CREATE_DATASET_GROUPS: group1,group2,group3
+          CREATE_DATASET_WITH_PID_GROUPS: "group2"
+          CREATE_DATASET_PRIVILEGED_GROUPS: "group3"
+          ACCESS_GROUPS_STATIC_VALUES: "ess"
+          PROPOSAL_GROUPS: "proposalingestor"
+          SAMPLE_GROUPS: ""
+          DATASET_CREATION_VALIDATION_ENABLED: true
+          DATASET_CREATION_VALIDATION_REGEX: "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-4[0-9A-Fa-f]{3}-[89ABab][0-9A-Fa-f]{3}-[0-9A-Fa-f]{12}$"
+          ES_HOST: https://localhost:9200
+          ES_USERNAME: elastic
+          ES_PASSWORD: duo-password
+          MONGODB_COLLECTION: Dataset
+          ES_MAX_RESULT: 210000
+          ES_FIELDS_LIMIT: 400000
+          ES_INDEX: "dataset"
+          ES_REFRESH: "wait_for"
+          STACK_VERSION: 8.8.2
+          CLUSTER_NAME: es-cluster
+          MEM_LIMIT: 4G
+
+        # Start mongo container and app before running api tests
+        run: |
+          docker-compose -f CI/ESS/docker-compose.api.yaml up --build -d
+          until curl -s -f -o /dev/null "localhost:27017"; do sleep 5; done
+          npm run start
+
+      - name: Generate openapi client
+        uses: openapi-generators/openapitools-generator-action@v1
+        with:
+          generator: python-pydantic-v1
+          openapi-file: openapi.json
+          config-file: .github/openapi/python-config.yml
+          command-args: "--ignore-file-override openapi/python/test**"
+
+  openapi_tests:
+    name: OpenAPI tests with ElasticSearch enabled
+    needs: [ generate_openapi ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Restore node_modules
+        id: cached-node-modules
+        uses: actions/cache@v4
+        with:
+          path: node_modules
+          key: node-modules-${{ hashFiles('package-lock.json') }}-${{ env.NODE_VERSION }}
+
+      - name: Start backend
+        env:
+          ELASTICSEARCH_ENABLED: yes
+          MONGODB_URI: mongodb://localhost:27017/scicat
+          EXPRESS_SESSION_SECRET: a_scicat_secret
+          JWT_SECRET: a_scicat_secret
+          PORT: 3000
+          HTTP_MAX_REDIRECTS: 5
+          HTTP_TIMEOUT: 5000
+          JWT_EXPIRES_IN: 3600
+          LOGGING_DISABLED: True
+          # NOTE: The SITE variable is set to PSI because of the PublishedData test.
+          # Testing /register endpoint expects to have it as PSI for that specific scenario.
+          # Old backend was modifying this at runtime but with NestJS we need to investigate little bit more if there is a bit better and more elegant solution.
+          SITE: PSI
+          PID_PREFIX: PID.SAMPLE.PREFIX
+          DOI_PREFIX: DOI.SAMPLE.PREFIX
+          METADATA_KEYS_RETURN_LIMIT: 100
+          METADATA_PARENT_INSTANCES_RETURN_LIMIT: 100
+          ADMIN_GROUPS: admin,ingestor
+          DELETE_GROUPS: archivemanager
+          CREATE_DATASET_GROUPS: group1,group2,group3
+          CREATE_DATASET_WITH_PID_GROUPS: "group2"
+          CREATE_DATASET_PRIVILEGED_GROUPS: "group3"
+          ACCESS_GROUPS_STATIC_VALUES: "ess"
+          PROPOSAL_GROUPS: "proposalingestor"
+          SAMPLE_GROUPS: ""
+          DATASET_CREATION_VALIDATION_ENABLED: true
+          DATASET_CREATION_VALIDATION_REGEX: "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-4[0-9A-Fa-f]{3}-[89ABab][0-9A-Fa-f]{3}-[0-9A-Fa-f]{12}$"
+          ES_HOST: https://localhost:9200
+          ES_USERNAME: elastic
+          ES_PASSWORD: duo-password
+          MONGODB_COLLECTION: Dataset
+          ES_MAX_RESULT: 210000
+          ES_FIELDS_LIMIT: 400000
+          ES_INDEX: "dataset"
+          ES_REFRESH: "wait_for"
+          STACK_VERSION: 8.8.2
+          CLUSTER_NAME: es-cluster
+          MEM_LIMIT: 4G
+
+        # Start mongo container and app before running api tests
+        run: |
+          cp functionalAccounts.json.test functionalAccounts.json
+          docker-compose -f CI/ESS/docker-compose.api.yaml up --build -d
+          until curl -s -f -o /dev/null "localhost:27017"; do sleep 5; done
+          npm run start &
+
+      - name: Test OpenAPI
+        run: |
+          cd openapi/python
+          python3 -m venv .venv
+          source .venv/bin/activate
+          python3 -m pip install --upgrade pip
+          pip install pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f test-requirements.txt ]; then pip install -r requirements.txt; fi
+          pytest

.gitignore

@@ -7,6 +7,10 @@ loggers.json
 # Configs
 .env
 
+# OpenAPI
+/openapi
+openapi.json
+
 # Logs
 logs
 *.log

CI/E2E/docker-compose-mongo.yaml

@@ -0,0 +1,11 @@
+version: "3.2"
+services:
+  mongodb:
+    image: bitnami/mongodb:latest
+    volumes:
+      - "mongodb_data:/bitnami"
+    ports:
+      - "27017:27017"
+volumes:
+  mongodb_data:
+    driver: local

openapi/python-nextgen/test/__init__.py

@@ -0,0 +1,37 @@
+import scicat_openapi_client as soc
+
+configuration = soc.Configuration(
+    host="http://localhost:3000"
+)
+api_client = soc.ApiClient(configuration)
+
+users = {
+    'admin': soc.CredentialsDto(username='admin', password='am2jf70TPNZsSan'),
+    'ingestor': soc.CredentialsDto(username='ingestor', password='aman'),
+    'proposalIngestor': soc.CredentialsDto(
+        username='proposalIngestor',
+        password='aman'),
+    'archiveManager': soc.CredentialsDto(
+        username='archiveManager',
+        password='aman'),
+    'user1': soc.CredentialsDto(
+        username='user2',
+        password='a609316768619f154ef58db4d847b75e'),
+    'user2': soc.CredentialsDto(
+        username='user2',
+        password='f522d1d715970073a6413474ca0e0f63'),
+    #
+    'nouser': soc.CredentialsDto(
+        username='nouser',
+        password='f522d1d715970073a6413474ca0e0f63'),
+}
+
+
+def login(api, user: str):
+    auth_api = soc.AuthApi(api_client)
+    login_dto = auth_api.auth_controller_login(users[user])
+    api.api_client.configuration.access_token = login_dto.access_token
+
+
+def logout(api):
+    api.api_client.configuration.access_token = ""