.gitlab-ci.yml

################################################################################
# Changes to this file requires approval from Labs. Please add a person from   #
# Labs as required approval to your MR if you have any changes.                #
################################################################################

# For the release steps to work, the following envionment variables have to set
# in the Gitlab UI:
# RELEASE_REGISTRY_USER
# RELEASE_REGISTRY_PASSWORD


stages:
  - lint
  - build
  - test
  - release
  - deploy


variables:
  # Project variables
  RELEASE_REGISTRY: docker.io
  RELEASE_REPORT_IMAGE: index.docker.io/magentaaps/os2datascanner-report
  RELEASE_ADMIN_IMAGE: index.docker.io/magentaaps/os2datascanner-admin
  RELEASE_ENGINE_IMAGE: index.docker.io/magentaaps/os2datascanner-engine
  RELEASE_API_IMAGE: index.docker.io/magentaaps/os2datascanner-api

  REPORT_IMAGE: ${CI_REGISTRY_IMAGE}/report:${CI_COMMIT_SHA}
  ADMIN_IMAGE: ${CI_REGISTRY_IMAGE}/admin:${CI_COMMIT_SHA}
  ENGINE_IMAGE: ${CI_REGISTRY_IMAGE}/engine:${CI_COMMIT_SHA}
  API_IMAGE: ${CI_REGISTRY_IMAGE}/api:${CI_COMMIT_SHA}


# Lint stage
#############

.lint-default: &lint-default
  stage: lint
  needs: []
  services: []
  tags:
    - docker


# We should lint our Python code. See: #44827.


Lint Dockerfiles:
  <<: *lint-default
  image: hadolint/hadolint:latest-alpine
  before_script:
    - apk add fd
  script:
    - echo "Running Hadolint, a Dockerfile linter"
    - echo "For more information about reported errors, visit the Hadolint wiki on https://github.com/hadolint/hadolint/wiki"
    # Run fd without exec arg to produce a list of files to be linted
    - fd Dockerfile
    # Run hadolint on every Dockerfile
    - fd Dockerfile --exec hadolint

Lint shell scripts:
  <<: *lint-default
  image: koalaman/shellcheck-alpine:latest
  before_script:
    - apk update
    - apk add fd
  script:
    - fd --extension sh --exec shellcheck


# Build stage
##############

.build-default: &build-default
  stage: build
  needs: []
  services: []
  image:
    name: gcr.io/kaniko-project/executor:v1.6.0-debug
    entrypoint: [""]
  tags:
    - docker

Build Report Application:
  <<: *build-default
  script:
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
    - /kaniko/executor
      --cache
      --context=$CI_PROJECT_DIR
      --dockerfile=$CI_PROJECT_DIR/docker/report/Dockerfile
      --destination=${REPORT_IMAGE}

Build Admin Application:
  <<: *build-default
  script:
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
    - /kaniko/executor
      --cache
      --context=$CI_PROJECT_DIR
      --dockerfile=$CI_PROJECT_DIR/docker/admin/Dockerfile
      --destination=${ADMIN_IMAGE}

Build Engine:
  <<: *build-default
  script:
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
    - /kaniko/executor
      --cache
      --context=$CI_PROJECT_DIR
      --dockerfile=$CI_PROJECT_DIR/docker/engine/Dockerfile
      --destination=${ENGINE_IMAGE}

Build API:
  <<: *build-default
  script:
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
    - /kaniko/executor
      --cache
      --context=$CI_PROJECT_DIR
      --dockerfile=$CI_PROJECT_DIR/docker/api/Dockerfile
      --destination=${API_IMAGE}

Build documentation:
  <<: *build-default
  image: keimlink/sphinx-doc:latex
  script:
    - make -C doc PAPER=a4 html latexpdf
    - mv doc/_build .
  artifacts:
    name: "$CI_JOB_STAGE-$CI_COMMIT_REF_NAME"
    paths:
      - _build


# Test stage
#############

.test-default: &test-default
  stage: test
  tags:
    - docker
    - network-per-build
  services:
    - postgres:12
    - rabbitmq:3.8-alpine
    - name: magentalabs/samba-test:1.1
      alias: samba
  variables:
    SMB_USER: os2
    SMB_PASSWD: 12345_rosebud_password_admin
    SMB_SHARE_NAME: general
    SMB_SHARE_PATH: ${CI_PROJECT_DIR}/src/os2datascanner/engine2/tests/data/engine2
    SMB_SHARE_BROWSABLE: "no"
    SMB_SHARE_READONLY: "yes"
    POSTGRES_DB: os2datascanner
    POSTGRES_USER: os2datascanner
    POSTGRES_PASSWORD: os2datascanner
    COVERAGE_FILE: /tmp/.coverage

Test Admin Application:
  extends: .test-default
  image: ${ADMIN_IMAGE}
  variables:
    OS2DS_ADMIN_USER_CONFIG_PATH: ${CI_PROJECT_DIR}/dev-environment/admin/test-settings.toml
  needs:
    - Build Admin Application
  script:
    - cd /code/src/os2datascanner/projects/admin
    - coverage run --omit=*/tests/* --source=os2datascanner.projects.admin -m django test
  after_script:
    - coverage html -d ${CI_PROJECT_DIR}/coverage-html
    - coverage xml -o ${CI_PROJECT_DIR}/junit.xml
    - coverage report
  coverage: '/^TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+(?:\.\d+)?\%)$/'
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/coverage-html
    reports:
      junit: $CI_PROJECT_DIR/junit.xml

Integration Test Engine:
  extends: .test-default
  image: ${ENGINE_IMAGE}
  variables:
    OS2DS_ENGINE_USER_CONFIG_PATH: ${CI_PROJECT_DIR}/dev-environment/engine/test-settings.toml
  needs:
    - Build Engine
  script:
    - pytest
      --junitxml $CI_PROJECT_DIR/junit.xml
      -p no:cacheprovider
      --color=yes
      src/os2datascanner/engine2/tests/integration/
  artifacts:
    when: always
    reports:
      junit: $CI_PROJECT_DIR/junit.xml

Unit Test Engine:
  stage: test
  tags:
    - docker
  image:
    name: ${ENGINE_IMAGE}
    entrypoint: [""]
  variables:
    OS2DS_ENGINE_USER_CONFIG_PATH: ${CI_PROJECT_DIR}/dev-environment/engine/test-settings.toml
  needs:
    - Build Engine
  script:
    - pytest
      -n 5
      --ignore-glob '*integration*'
      --junitxml $CI_PROJECT_DIR/junit.xml
      -p no:cacheprovider
      --color=yes
      src/os2datascanner/engine2/tests/
  artifacts:
    when: always
    reports:
      junit: $CI_PROJECT_DIR/junit.xml

Test Report Application:
  extends: .test-default
  image: ${REPORT_IMAGE}
  variables:
    OS2DS_REPORT_USER_CONFIG_PATH: ${CI_PROJECT_DIR}/dev-environment/report/test-settings.toml
  needs:
    - Build Report Application
  script:
    - coverage run --omit=*/tests/* --source=os2datascanner.projects.report -m django test os2datascanner.projects.report.tests
  after_script:
    - coverage html -d ${CI_PROJECT_DIR}/coverage-html
    - coverage xml -o ${CI_PROJECT_DIR}/junit.xml
    - coverage report
  coverage: '/^TOTAL\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+(?:\.\d+)?\%)$/'
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/coverage-html
    reports:
      junit: $CI_PROJECT_DIR/junit.xml

Run Engine notice scripts:
  stage: test
  tags:
    - docker
  services: []
  image: ${ENGINE_IMAGE}
  needs:
    - Build Engine
  variables:
    OS2DS_ENGINE_USER_CONFIG_PATH: ${CI_PROJECT_DIR}/dev-environment/engine/test-settings.toml
    # ensure entrypoint.sh is running without trying to connect to DB/RabbitMQ/
    # This cannot be yes/no or something `yaml` interprets as a boolean
    BARE_MODE: yup
  script:
    - python -m unittest discover -s src/os2datascanner/engine2/tests/ --pattern "notice*.py"
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"


# Release stage
###############

.release-default: &release-default
  stage: release
  needs:
    - Lint Dockerfiles
    - Lint shell scripts
    - Build Report Application
    - Build Admin Application
    - Build Engine
    - Build API
    - Build documentation
    - Test Admin Application
    - Integration Test Engine
    - Unit Test Engine
    - Test Report Application
  image: alpine
  variables:
    GIT_STRATEGY: none # We do not need the source code
  tags:
    - docker

# Rolling rc release:
.release-rc: &release-rc
  <<: *release-default
  rules:
    - if: '$CI_COMMIT_REF_NAME == "development" && $CI_PIPELINE_SOURCE != "schedule"'
  script:
    - apk add skopeo
    - skopeo copy
      --src-creds=${CI_REGISTRY_USER}:${CI_BUILD_TOKEN}
      --dest-creds=${RELEASE_REGISTRY_USER}:${RELEASE_REGISTRY_PASSWORD}
      "docker://${IMAGE_FROM}"
      "docker://${IMAGE_TO}:dev"

Release Report Application Candidate:
  <<: *release-rc
  before_script:
    - export IMAGE_FROM="${REPORT_IMAGE}"
    - export IMAGE_TO="${RELEASE_REPORT_IMAGE}"

Release Admin Application Candidate:
  <<: *release-rc
  before_script:
    - export IMAGE_FROM="${ADMIN_IMAGE}"
    - export IMAGE_TO="${RELEASE_ADMIN_IMAGE}"

Release Engine Candidate:
  <<: *release-rc
  before_script:
    - export IMAGE_FROM="${ENGINE_IMAGE}"
    - export IMAGE_TO="${RELEASE_ENGINE_IMAGE}"

Release API Candidate:
  <<: *release-rc
  before_script:
    - export IMAGE_FROM="${API_IMAGE}"
    - export IMAGE_TO="${RELEASE_API_IMAGE}"

# Release pinned pre-release:
# Separate from "Versioned release" (.release) because we don't want to push to :latest tags
.release-pinned-rc: &release-pinned-rc
  <<: *release-default
  only:
    variables:
      # Matches SemVer 2.0.0 with pre-release group. Ex. 2.3.4-rc, 2.3.4-rc2 or 2.3.4-prerelease1, but not 2.3.4
      - $CI_COMMIT_TAG =~ /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)){1}(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
  script:
    - apk add skopeo
    - skopeo copy
      --src-creds=${CI_REGISTRY_USER}:${CI_BUILD_TOKEN}
      --dest-creds=${RELEASE_REGISTRY_USER}:${RELEASE_REGISTRY_PASSWORD}
      "docker://${IMAGE_FROM}"
      "docker://${IMAGE_TO}:${CI_COMMIT_TAG}"

Release Report Application pinned Candidate:
  <<: *release-pinned-rc
  before_script:
    - export IMAGE_FROM="${REPORT_IMAGE}"
    - export IMAGE_TO="${RELEASE_REPORT_IMAGE}"

Release Admin Application pinned Candidate:
  <<: *release-pinned-rc
  before_script:
    - export IMAGE_FROM="${ADMIN_IMAGE}"
    - export IMAGE_TO="${RELEASE_ADMIN_IMAGE}"

Release Engine pinned Candidate:
  <<: *release-pinned-rc
  before_script:
    - export IMAGE_FROM="${ENGINE_IMAGE}"
    - export IMAGE_TO="${RELEASE_ENGINE_IMAGE}"

Release API pinned Candidate:
  <<: *release-pinned-rc
  before_script:
    - export IMAGE_FROM="${API_IMAGE}"
    - export IMAGE_TO="${RELEASE_API_IMAGE}"

# Versioned release:
.release: &release
  <<: *release-default
  rules:
      # Matches <version core> from SemVer 2.0.0 BNF grammar. Ex. 2.3.4, but not 2.3.4-rc
      - if: $CI_COMMIT_TAG =~ /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/
  script:
    - apk add skopeo
    - skopeo copy
      --src-creds=${CI_REGISTRY_USER}:${CI_BUILD_TOKEN}
      --dest-creds=${RELEASE_REGISTRY_USER}:${RELEASE_REGISTRY_PASSWORD}
      "docker://${IMAGE_FROM}"
      "docker://${IMAGE_TO}:${CI_COMMIT_TAG}"
    - skopeo copy
      --src-creds=${CI_REGISTRY_USER}:${CI_BUILD_TOKEN}
      --dest-creds=${RELEASE_REGISTRY_USER}:${RELEASE_REGISTRY_PASSWORD}
      "docker://${IMAGE_FROM}"
      "docker://${IMAGE_TO}:latest"

Release Report Application:
  <<: *release
  before_script:
    - export IMAGE_FROM="${REPORT_IMAGE}"
    - export IMAGE_TO="${RELEASE_REPORT_IMAGE}"

Release Admin Application:
  <<: *release
  before_script:
    - export IMAGE_FROM="${ADMIN_IMAGE}"
    - export IMAGE_TO="${RELEASE_ADMIN_IMAGE}"

Release Engine:
  <<: *release
  before_script:
    - export IMAGE_FROM="${ENGINE_IMAGE}"
    - export IMAGE_TO="${RELEASE_ENGINE_IMAGE}"

Release API:
  <<: *release
  before_script:
    - export IMAGE_FROM="${API_IMAGE}"
    - export IMAGE_TO="${RELEASE_API_IMAGE}"

# Deploy stage
##############

Deploy test:
  # Automatic deployment of development branch to
  # https://test.os2datascanner.dk/ and https://test-admin.os2datascanner.dk/
  stage: deploy
  image: magentalabs/pepper:latest
  services: []
  rules:
    - if: '$CI_COMMIT_REF_NAME == "development" && $CI_PIPELINE_SOURCE != "schedule"'
  dependencies: []
  tags:
    - deploy
  needs:
    - Release Report Application Candidate
    - Release Admin Application Candidate
    - Release Engine Candidate
    - Release API Candidate
  variables:
    SALTAPI_URL: https://ctrldev1.magenta-aps.dk/api/
    SALTAPI_EAUTH: file
    GIT_STRATEGY: none
  script:
    - pepper --client runner --fail-any state.orchestrate os2ds.service-test
  environment:
    name: test
    url: https://test-admin.os2datascanner.dk/