GitHub Actions: Add unit tests workflow (DefectDojo#3063)

.github/workflows/flake8-your-pr.yml → .github/workflows/flake8.yml

@@ -1,6 +1,15 @@
-name: Flake8 your PR
-# run on pull_request_target instead of just pull_request as we need write access to update the status check
-on: [pull_request_target]
+name: Flake8
+# pull requests: 
+#      run on pull_request_target instead of just pull_request as we need write access to update the status check
+# dev, master:
+#      run on every push, which is when something gets merged also
+on: 
+  pull_request_target:
+  push:
+    branches:
+      - master
+      - develop
+
 jobs:
   flake8-your-pr:
     runs-on: ubuntu-latest

.github/workflows/integration-tests.yml

@@ -1,6 +1,15 @@
 name: Integration tests
-# run on pull_request_target instead of just pull_request as we need write access to update the status check
-on: [pull_request_target]
+# pull requests: 
+#      run on pull_request_target instead of just pull_request as we need write access to update the status check
+# dev, master:
+#      run on every push, which is when something gets merged also
+on: 
+  pull_request_target:
+  push:
+    branches:
+      - master
+      - develop
+
 jobs:
   integration_test:
     runs-on: ubuntu-latest

.github/workflows/unit-tests.yml

@@ -1,6 +1,15 @@
 name: Unit tests
-# run on pull_request_target instead of just pull_request as we need write access to update the status check
-on: [pull_request_target]
+# pull requests: 
+#      run on pull_request_target instead of just pull_request as we need write access to update the status check
+# dev, master:
+#      run on every push, which is when something gets merged also
+on: 
+  pull_request_target:
+  push:
+    branches:
+      - master
+      - develop
+
 jobs:
   integration_test:
     runs-on: ubuntu-latest

docker-compose.override.unit_tests.yml

@@ -7,7 +7,7 @@ services:
       dockerfile: Dockerfile.busybox
     image: defectdojo/defectdojo-busybox:${NGINX_VERSION:-latest}
   uwsgi:
-    entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/entrypoint-unit-tests-devDocker.sh']
+    entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/app/docker/entrypoint-unit-tests-devDocker.sh']
     volumes:
       - '.:/app:z'
     environment:

docker-compose.override.unit_tests_cicd.yml

@@ -0,0 +1,37 @@
+---
+version: '3.7'
+services:
+  nginx:
+    build: 
+      context: ./
+      dockerfile: Dockerfile.busybox
+    image: defectdojo/defectdojo-busybox:${NGINX_VERSION:-latest}
+  uwsgi:
+    entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/app/docker/entrypoint-unit-tests.sh']
+    volumes:
+      - '.:/app:z'
+    environment:
+      DD_DEBUG: 'True'
+      DD_TEST_DATABASE_NAME: ${DD_TEST_DATABASE_NAME:-test_defectdojo}
+      DD_DATABASE_NAME: ${DD_TEST_DATABASE_NAME:-test_defectdojo}
+  celerybeat:
+    environment:
+      DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-mysql://defectdojo:defectdojo@mysql:3306/test_defectdojo}
+  celeryworker:
+    environment:
+      DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-mysql://defectdojo:defectdojo@mysql:3306/test_defectdojo}
+  initializer:
+    environment:
+      DD_INITIALIZE: 'false'
+  mysql:
+    ports:
+      - target: 3306
+        published: 3306
+        protocol: tcp
+        mode: host
+    environment:
+      MYSQL_DATABASE: ${DD_TEST_DATABASE_NAME:-test_defectdojo}
+    volumes:
+       - defectdojo_data_unittest:/var/lib/mysql
+volumes:
+  defectdojo_data_unittest: {}

docker/entrypoint-unit-tests-devDocker.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
-# Run available unittests with a simple setup
-umask 0002
+# Run available unittests with a setup for local dev:
+# - Make migrations and apply any needed changes
+# - Leave container up after running tests to allow debugging, rerunning tests, etc.
+set -x
+set -e
+set -v
 
 cd /app
 # Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py)

docker/entrypoint-unit-tests.sh

@@ -1,9 +1,22 @@
 #!/bin/sh
-# Run available unittests with a simple setup
+# Run available unittests with a setup for CI/CD:
+# - Fail if migrations are not created
+# - Exit container after running tests to allow exit code to propagate as test result
+set -x
+set -e
+set -v
 
 cd /app
+# Unset the database URL so that we can force the DD_TEST_DATABASE_NAME (see django "DATABASES" configuration in settings.dist.py)
+unset DD_DATABASE_URL
 
-./manage.py makemigrations --no-input --check --dry-run --verbosity 3 || {
+TARGET_SETTINGS_FILE=dojo/settings/settings.py
+if [ ! -f ${TARGET_SETTINGS_FILE} ]; then
+  echo "Creating settings.py"
+  cp dojo/settings/settings.dist.py dojo/settings/settings.py
+fi
+
+python3 manage.py makemigrations --no-input --check --dry-run --verbosity 3 || {
     cat <<-EOF
 
 ********************************************************************************
@@ -22,6 +35,6 @@ EOF
     exit 1
 }
 
-./manage.py migrate
+python3 manage.py migrate
 
-exec ./manage.py test dojo.unittests -v 2
+python3 manage.py test dojo.unittests -v 3 --no-input

docker/setEnv.sh

@@ -6,6 +6,7 @@ target_dir="${0%/*}/.."
 override_link='docker-compose.override.yml'
 override_file_dev='docker-compose.override.dev.yml'
 override_file_unit_tests='docker-compose.override.unit_tests.yml'
+override_file_unit_tests_cicd='docker-compose.override.unit_tests_cicd.yml'
 override_file_integration_tests='docker-compose.override.integration_tests.yml'
 override_file_ptvsd='docker-compose.override.ptvsd.yml'
 
@@ -88,6 +89,19 @@ function set_unit_tests {
     fi
 }
 
+function set_unit_tests_cicd {
+    get_current
+    if [ "${current_env}" != unit_tests_cicd ]
+    then
+        rm -f ${override_link}
+        ln -s ${override_file_unit_tests_cicd} ${override_link}
+        docker-compose down
+        echo "Now using 'unit_tests_cicd' configuration."
+    else
+        echo "Already using 'unit_tests_cicd' configuration."
+    fi
+}
+
 function set_integration_tests {
     get_current
     if [ "${current_env}" != integration_tests ]
@@ -117,7 +131,7 @@ function set_ptvsd {
 # Change directory to allow working with relative paths.
 cd ${target_dir}
 
-if [ ${#} -eq 1 ] && [[ 'dev unit_tests integration_tests release ptvsd' =~ "${1}" ]]
+if [ ${#} -eq 1 ] && [[ 'dev unit_tests unit_tests_cicd integration_tests release ptvsd' =~ "${1}" ]]
 then
     set_"${1}"
 else