merge with release-jobs

.env.example

@@ -23,6 +23,7 @@ PUBLIC_URL_PREFIX="https://doi.esss.se/detail/"
 PORT=3000
 RABBITMQ_ENABLED=<"yes"|"no">
 RABBITMQ_HOSTNAME="localhost"
+RABBITMQ_PORT=5672
 RABBITMQ_USERNAME="rabbitmq"
 RABBITMQ_PASSWORD="rabbitmq"
 REGISTER_DOI_URI="https://mds.test.datacite.org/doi"
@@ -79,3 +80,4 @@ ES_PASSWORD="duo-password"
 ES_REFRESH=<"wait_for"|"false">
 
 LOGGERS_CONFIG_FILE="loggers.json"
+PROPOSAL_TYPES_FILE="proposalTypes.json"

.github/ISSUE_TEMPLATE/issue-template.md

@@ -1,25 +1,23 @@
 ---
 name: Issue template
-about: Describe this issue purpose here.
-
+about: Generic template for backend issues
 ---
 
-## Issue Name (title)
-Please enter the title for this github issue
+## Summary
+<!-- Description of the issue -->
 
-### Summary
-Description of the issue
+## Steps to Reproduce
+<!-- Describe how to reproduce the issue -->
 
-### Steps to Reproduce
-Describe how to reproduce the issue
+## Current Behaviour
+<!-- Describe the current behavior if it is relevant -->
 
-### Current Behaviour
-Describe the current behavior if it is relevant
+## Expected Behaviour
+<!-- Describe the expected behavior if it is relevant -->
 
-### Expected Behaviour
-Describe the expected behavior if it is relevant
+## Details
+<!-- Provide all the details related to this issue. Also include screenshots, wireframes or mockups if they help.
+Link related issues in the frontend or other repositories if relevant. -->
 
-### Details
-Provide all the details related to this issue. Also include screenshots, wireframes or mockups if they help.
 
-__PS__: make sure to add the proper tags to the issue
+<!-- P.S. Make sure to add the proper tags to the issue -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,31 +1,38 @@
-## Description
+<!--
+Follow semantic-release guidelines for the PR title, which is used in the changelog.
 
-Short description of the pull request
+Title should follow the format `<type>(<scope>): <subject>`, where
+- Type is one of: build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test|BREAKING CHANGE
+- Scope (optional) describes the place of the change (eg a particular milestone) and is usually omitted
+- subject should be a non-capitalized one-line description in present imperative tense and not ending with a period
 
-## Motivation
+See https://github.com/angular/angular.js/blob/master/DEVELOPERS.md#-git-commit-guidelines for more details.
+-->
 
-Background on use case, changes needed
+## Description
+<!-- Short description of the pull request -->
 
-## Fixes:
-Please provide a list of the fixes implemented by this PR
+## Motivation
+<!-- Background on use case, changes needed -->
 
-* Items added
+## Fixes
+<!-- Please provide a list of the issues fixed by this PR -->
+
+* Bug fixed (#X)
 
 ## Changes:
-Please provide a list of the changes implemented by this PR
+<!-- Please provide a list of the changes implemented by this PR -->
 
 * changes made
 
 ## Tests included
 
 - [ ] Included for each change/fix?
-- [ ] Passing? (Merge will not be approved unless this is checked) 
+- [ ] Passing? <!-- Merge will not be approved unless tests pass -->
 
 ## Documentation
-- [ ] swagger documentation updated \[required\]
-- [ ] official documentation updated \[nice-to-have]
+- [ ] swagger documentation updated (required for API changes)
+- [ ] official documentation updated
 
 ### official documentation info
-If you have updated the official documentation, please provide PR # and URL of the pages where the updates are included
-
-
+<!-- If you have updated the official documentation, please provide PR # and URL of the updated pages -->

.github/openapi/python-config.json

@@ -1,5 +1,5 @@
 {
   "generatorName": "python",
-  "packageName": "scicat-sdk-py",
-  "projectName": "scicat-sdk-py"
+  "packageName": "scicat_sdk_py",
+  "projectName": "scicat_sdk_py"
 }

.github/openapi/python-pydantic-v1-config.json

@@ -1,5 +1,5 @@
 {
   "generatorName": "python-pydantic-v1",
-  "packageName": "scicat-sdk-pydantic",
-  "projectName": "scicat-sdk-pydantic"
+  "packageName": "scicat_sdk_pydantic",
+  "projectName": "scicat_sdk_pydantic"
 }

.github/pr-title-checker-config.json

@@ -9,7 +9,7 @@
   },
   "MESSAGES": {
     "success": "All OK",
-    "failure": "PR title not following the semantic-release guidelines. Please check https://github.com/angular/angular.js/blob/master/DEVELOPERS.md#-git-commit-guidelines for more information.",
+    "failure": "PR title not following the semantic-release guidelines (<type>(<scope>): <subject>). Please check https://github.com/angular/angular.js/blob/master/DEVELOPERS.md#-git-commit-guidelines for more information.",
     "notice": ""
   }
 }

.github/workflows/deploy.yml

@@ -38,12 +38,3 @@ jobs:
           platforms: linux/amd64,linux/arm64/v8
           push: true
           tags: ${{ steps.meta.outputs.tags }}
-
-      - name: Trigger ESS pipeline
-        uses: swapActions/trigger-swap-deployment@v1
-        with:
-          repository: ${{ github.event.repository.name }}
-          environment: develop
-          gh-trigger-url: ${{ secrets.GITLAB_TRIGGER_URL }}
-          gh-token: ${{ secrets.GITLAB_TRIGGER_TOKEN }}
-          image-tag: ${{ github.sha }}

.github/workflows/release-and-publish-sdk.yml

@@ -193,6 +193,7 @@ jobs:
         run: |
           npm install
           npm run build
+          cd dist
           npm publish --access public
         working-directory: ./sdk/typescript-angular/
         env:

.github/workflows/test.yml

@@ -155,7 +155,7 @@ jobs:
           ACCESS_GROUPS_STATIC_VALUES: "ess"
           CREATE_JOB_GROUPS: group1,group2
           UPDATE_JOB_GROUPS: group1
-          DELETE_JOB_GROUPS: "archivemanager"
+          DELETE_JOB_GROUPS: "archivemanager,admin"
           PROPOSAL_GROUPS: "proposalingestor"
           SAMPLE_PRIVILEGED_GROUPS: "sampleingestor"
           SAMPLE_GROUPS: "group1"
@@ -172,11 +172,12 @@ jobs:
           STACK_VERSION: 8.8.2
           CLUSTER_NAME: es-cluster
           MEM_LIMIT: 4G
-          JOB_CONFIGURATION_FILE: test/config/jobconfig.json
+          JOB_CONFIGURATION_FILE: test/config/jobconfig.yaml
 
         # Start mongo container and app before running api tests
         run: |
           cp CI/ESS/docker-compose.api.yaml docker-compose.yaml
           cp functionalAccounts.json.test functionalAccounts.json
+          cp proposalTypes.example.json proposalTypes.json
           docker compose up --build -d
           npm run test:api

.gitignore

@@ -2,8 +2,10 @@
 /dist
 /node_modules
 functionalAccounts.json
+proposalTypes.json
 loggers.json
 jobConfig.json
+jobConfig.yaml
 
 # Configs
 .env
@@ -44,4 +46,3 @@ lerna-debug.log*
 /mongodb_data
 test-data/*
 .vscode*
-

README.md

@@ -40,27 +40,29 @@ Thank you for your interest in contributing to our project!
 3. Add _.env_ file to project root folder. See [Environment variables](#environment-variables).
 4. _Optional_ Add [functionalAccounts.json](#local-user-accounts) file to project root folder to create local users.
 5. _Optional_ Add [loggers.json](#loggers-configuration) file to the root folder and configure multiple loggers.
-6. `npm run start:dev`
-7. Go to http://localhost:3000/explorer to get an overview of available endpoints and database schemas.
-8. To be able to run the e2e tests with the same setup as in the Github actions you will need to run `npm run  prepare:local` and after that run `npm run start:dev`. This will start all needed containers and copy some configuration to the right place.
+6. _Optional_ Add [proposalTypes.json](#prpopsal-types-configuration) file to the root folder and configure the proposal types.
+7. `npm run start:dev`
+8. Go to http://localhost:3000/explorer to get an overview of available endpoints and database schemas.
+9. To be able to run the e2e tests with the same setup as in the Github actions you will need to run `npm run  prepare:local` and after that run `npm run start:dev`. This will start all needed containers and copy some configuration to the right place.
 
 ## Develop in a container using the docker-compose.dev file
 
 1. `git clone https://github.com/SciCatProject/scicat-backend-next.git`
 2. docker-compose -f docker-compose.dev.yaml up -d
 3. _Optional_ Mount [functionalAccounts.json](#local-user-accounts) file to a volume in the container to create local users.
 4. _Optional_ Mount [loggers.json](#loggers-configuration) file to a volume in the container to configure multiple loggers.
-5. _Optional_ change the container env variables
-6. Attach to the container
-7. `npm run start:dev`
-8. Go to http://localhost:3000/explorer to get an overview of available endpoints and database schemas.
+5. _Optional_ Mount [proposalTypes.json](#prpopsal-types-configuration) file to a volume in the container to configure the proposal types.
+6. _Optional_ change the container env variables
+7. Attach to the container
+8. `npm run start:dev`
+9. Go to http://localhost:3000/explorer to get an overview of available endpoints and database schemas.
 
 ## Test the app
 
 1. **Running the unit tests:** `npm run test`
 2. **Running the e2e(api) tests:**
 
-- First of all run `npm run prepare:local` to prepare the local environment for starting
+- First of all run `npm run prepare:local` to prepare the local docker environment for starting
 - After that run `npm run test:api` which will start the backend locally and run both `jest` and `mocha` e2e(api) tests.
 - [Optional] If you want to run only the mocha tests you will need to start the backend locally with `npm run start` and then use `npm run test:api:mocha`
 - [Optional] If you want to run only the jest tests you can use `npm run test:api:jest`
@@ -96,6 +98,12 @@ Providing a file called _loggers.json_ at the root of the project, locally or in
 
 The `loggers.json.example` file in the root directory showcases the example of configuration structure for the one or multiple loggers. `logger.service.ts` file contains the configuration handling process logic, and `src/loggers/loggingProviders/grayLogger.ts` includes actual usecase of grayLogger.
 
+### Prpopsal types configuration
+
+Providing a file called _proposalTypes.json_ at the root of the project, locally or in the container, will be automatically loaded into the application configuration service under property called `proposalTypes` and used for validation against proposal creation and update.
+
+The `proposalTypes.json.example` file in the root directory showcases the example of configuration structure for proposal types.
+
 ## Environment variables
 
 Valid environment variables for the .env file. See [.env.example](/.env.example) for examples value formats.
@@ -176,7 +184,9 @@ Valid environment variables for the .env file. See [.env.example](/.env.example)
 | `ES_FIELDS_LIMIT` | number | | The total number of fields in an index. | 1000 |
 | `ES_REFRESH` | string | | If set to `wait_for`, Elasticsearch will wait till data is inserted into the specified index before returning a response. | false |
 | `LOGGERS_CONFIG_FILE` | string | | The file name for loggers configuration, located in the project root directory. | "loggers.json" |
+| `PROPOSAL_TYPES_FILE` | string | | The file name for proposal types configuration, located in the project root directory. | "proposalTypes.json" |
 | `SWAGGER_PATH` | string | Yes | swaggerPath is the path where the swagger UI will be available| "explorer"|
+| `MAX_FILE_UPLOAD_SIZE` | string | Yes | Maximum allowed file upload size | "16mb"|
 
 ## Migrating from the old SciCat Backend
 

jobConfig.example.yaml

@@ -0,0 +1,43 @@
+configVersion: v1.0 2024-03-01 6f3f38
+jobs:
+  - jobType: archive
+    create:
+      auth: "#all"
+      actions:
+        - actionType: log
+        - actionType: url
+          url: http://localhost:3000/api/v3/health?jobid={{id}}
+          headers:
+            accept: application/json
+        - actionType: rabbitmq
+          exchange: jobs.write
+          queue: client.jobs.write
+          key: jobqueue
+        - actionType: email
+          auth:
+            user: user
+            password: password
+          to: "{{contactEmail}}"
+          from: from
+          subject: "[SciCat] Your {{type}} job was submitted successfully"
+          bodyTemplateFile: src/common/email-templates/job-template-simplified.html
+    statusUpdate:
+      auth: archivemanager
+      actions:
+        - actionType: rabbitmq
+          exchange: jobs.write
+          queue: client.jobs.write
+          key: jobqueue
+  - jobType: public
+    create:
+      auth: "#all"
+      actions:
+        - actionType: validate
+          request:
+            jobParams.datasetList[*]:
+              type: object
+              required:
+                - pid
+                - files
+    statusUpdate:
+      auth: "#all"

migrations/20241113130700-proposal-type.js

@@ -0,0 +1,12 @@
+module.exports = {
+  async up(db, client) {
+    db.collection("Proposal").updateMany(
+      { type: { $exists: false } },
+      { $set: { type: "Default Proposal" } },
+    );
+  },
+
+  async down(db, client) {
+    db.collection("Proposal").updateMany({}, { $unset: { type: 1 } });
+  },
+};