move unittests outside dojo folder (DefectDojo#5527)

* move unittests outside dojo folder

* change base class

* cleanup

* cleanup

* cleanup

* cleanup

* fixes

* cleanup

* refactor

* cleanup

* cleanup

* cleanup

* fixes

* fixes

* fixes

.github/labeler.yml

@@ -11,7 +11,7 @@ docker:
 - dojo/db_migrations/*
 
 unittests:
-- dojo/unittests/**/*
+- unittests/**/*
 
 integration_tests:
 - tests/**/*

Dockerfile.django

@@ -5,11 +5,12 @@
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.8.12-slim-buster@sha256:7e732593f25983fa7d4b2d54444be4a785f2397c1cf2f815d2e3638337eee012 as build
+FROM python:3.8.12-slim-buster@sha256:7e732593f25983fa7d4b2d54444be4a785f2397c1cf2f815d2e3638337eee012 as base
+FROM base as build
 WORKDIR /app
 RUN \
   apt-get -y update && \
-  apt-get -y install \
+  apt-get -y install --no-install-recommends \
     build-essential \
     dnsutils \
     default-mysql-client \
@@ -25,7 +26,7 @@ RUN \
 COPY requirements.txt ./
 RUN pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 
-FROM python:3.8.12-slim-buster@sha256:7e732593f25983fa7d4b2d54444be4a785f2397c1cf2f815d2e3638337eee012
+FROM base as django
 WORKDIR /app
 ARG uid=1001
 ARG appuser=defectdojo
@@ -136,3 +137,6 @@ ENV \
   DD_DJANGO_METRICS_ENABLED="False"
 RUN mkdir -p media && mkdir -p media/threat && chown -R ${uid} media
 ENTRYPOINT ["/entrypoint-uwsgi.sh"]
+
+FROM django as django-unittests
+COPY unittests/ ./unittests/

Dockerfile.nginx

@@ -32,7 +32,7 @@ ENV \
   node="nodejs"
 RUN \
   apt-get -y update && \
-  apt-get -y install apt-transport-https ca-certificates curl wget && \
+  apt-get -y install --no-install-recommends apt-transport-https ca-certificates curl wget && \
   curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add --no-tty - && \
   echo 'deb https://deb.nodesource.com/node_14.x buster main' > /etc/apt/sources.list.d/nodesource.list && \
   echo 'deb-src https://deb.nodesource.com/node_14.x buster main' >> /etc/apt/sources.list.d/nodesource.list && \
@@ -42,7 +42,7 @@ RUN \
   wget https://github.com/yarnpkg/yarn/releases/download/v1.22.10/yarn_1.22.10_all.deb && \
   dpkg -i yarn_1.22.10_all.deb && \
   echo "$(yarn --version)" && \
-  apt-get -y install nodejs && \
+  apt-get -y install --no-install-recommends nodejs && \
   echo "$(node --version)" && \
   apt-get clean && \
   rm yarn_1.22.10_all.deb && \

docker-compose.override.unit_tests.yml

@@ -7,6 +7,8 @@ services:
     volumes:
       - defectdojo_media_unittest:/usr/share/nginx/html/media
   uwsgi:
+    build:
+      target: django-unittests
     entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/app/docker/entrypoint-unit-tests-devDocker.sh']
     volumes:
       - '.:/app:z'

docker-compose.override.unit_tests_cicd.yml

@@ -7,6 +7,8 @@ services:
     volumes:
       - defectdojo_media_unittest:/usr/share/nginx/html/media
   uwsgi:
+    build:
+      target: django-unittests
     entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/app/docker/entrypoint-unit-tests.sh']
     volumes:
       - '.:/app:z'

docker-compose.yml

@@ -1,8 +1,8 @@
 # This docker-compose.yml file  is fully functional to evaluate DefectDojo
-# in your local environment. 
+# in your local environment.
 #
 # Although Docker Compose is one of the supported installation methods to
-# deploy a containerized DefectDojo in a production environment, the 
+# deploy a containerized DefectDojo in a production environment, the
 # docker-compose.yml file is not intended for production use without first
 # customizing it to your particular situation.
 ---
@@ -32,6 +32,7 @@ services:
     build:
       context: ./
       dockerfile: Dockerfile.django
+      target: django
     image: "defectdojo/defectdojo-django:${DJANGO_VERSION:-latest}"
     depends_on:
       - mysql

docker/entrypoint-unit-tests-devDocker.sh

@@ -45,17 +45,17 @@ EOF
 
 echo "Swagger Schema Tests - Broken"
 echo "------------------------------------------------------------"
-python3 manage.py test dojo.unittests -v 3 --keepdb --no-input --tag broken && true
+python3 manage.py test unittests -v 3 --keepdb --no-input --tag broken && true
 
 echo "Unit Tests"
 echo "------------------------------------------------------------"
-python3 manage.py test dojo.unittests -v 3 --keepdb --no-input --exclude-tag broken
+python3 manage.py test unittests -v 3 --keepdb --no-input --exclude-tag broken
 
 # you can select a single file to "test" unit tests
-# python3 manage.py test dojo.unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser --keepdb -v 3
+# python3 manage.py test unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser --keepdb -v 3
 
 # or even a single method
-# python3 manage.py test dojo.unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser.test_npm_audit_parser_many_vuln_npm7 --keepdb -v 3
+# python3 manage.py test unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser.test_npm_audit_parser_many_vuln_npm7 --keepdb -v 3
 
 echo "End of tests. Leaving the container up"
 tail -f /dev/null

docker/entrypoint-unit-tests.sh

@@ -66,12 +66,12 @@ EOF
 python3 manage.py migrate
 
 # --parallel fails on GitHub Actions
-#python3 manage.py test dojo.unittests -v 3 --no-input --parallel
+#python3 manage.py test unittests -v 3 --no-input --parallel
 
 echo "Swagger Schema Tests - Broken"
 echo "------------------------------------------------------------"
-python3 manage.py test dojo.unittests -v 3 --keepdb --no-input --tag broken && true
+python3 manage.py test unittests -v 3 --keepdb --no-input --tag broken && true
 
 echo "Unit Tests"
 echo "------------------------------------------------------------"
-python3 manage.py test dojo.unittests -v 3 --keepdb --no-input --exclude-tag broken
+python3 manage.py test unittests -v 3 --keepdb --no-input --exclude-tag broken

docker/unit-tests.sh

@@ -3,4 +3,4 @@
 cd /app
 python manage.py makemigrations dojo
 python manage.py migrate
-python manage.py test dojo.unittests -v 2
+python manage.py test unittests -v 2

docs/content/en/contributing/how-to-write-a-parser.md

@@ -46,7 +46,7 @@ Parser are loaded dynamicaly with a factory pattern. To have your parser loaded
 1. your parser **MUST** be in a sub-module of module `dojo.tools`
    - ex: `dojo.tools.my_tool.parser` module
 2. your parser **MUST** be a class in this sub-module.
-   - ex: `dojo.tools.my_tool.parser.MyToolParser` 
+   - ex: `dojo.tools.my_tool.parser.MyToolParser`
 3. The name of this class **MUST** be the Python module name without underscores and with `Parser` suffix.
    - ex: `dojo.tools.my_tool.parser.MyToolParser`
 4. This class **MUST** have an empty constructor or no constructor
@@ -113,7 +113,7 @@ Here is a list of advise that will make your parser future proof.
 ### Do not parse URLs by hand
 
 We use 2 modules to handle endpoints:
- - `hyperlink` 
+ - `hyperlink`
  - `dojo.models` with a specific class to handle processing around URLs to create endpoints `Endpoint`.
 
 All the existing parser use the same code to parse URL and create endpoints.
@@ -141,7 +141,7 @@ Very bad example:
 Parsers may have many fields, out of which many of them may be optional.
 It better to not set attribute if you don't have data instead of filling with values like `NA`, `No data` etc...
 
-Check class `dojo.models.Finding` 
+Check class `dojo.models.Finding`
 
 ### Data could be missing in the source report
 
@@ -259,17 +259,17 @@ MYSQL> flush privileges;
 This local command will launch the unit test for your new parser
 
 {{< highlight bash >}}
-$ docker-compose exec uwsgi bash -c 'python manage.py test dojo.unittests.tools.<your_unittest_py_file>.<main_class_name> -v2'
+$ docker-compose exec uwsgi bash -c 'python manage.py test unittests.tools.<your_unittest_py_file>.<main_class_name> -v2'
 {{< /highlight >}}
 
 Example for the blackduck hub parser:
 
 {{< highlight bash >}}
-$ docker-compose exec uwsgi bash -c 'python manage.py test dojo.unittests.tools.test_blackduck_csv_parser.TestBlackduckHubParser -v2'
+$ docker-compose exec uwsgi bash -c 'python manage.py test unittests.tools.test_blackduck_csv_parser.TestBlackduckHubParser -v2'
 {{< /highlight >}}
 
 {{% alert title="Information" color="info" %}}
-If you want to run all unit tests, simply run `$ docker-compose exec uwsgi bash -c 'python manage.py test dojo.unittests -v2'`
+If you want to run all unit tests, simply run `$ docker-compose exec uwsgi bash -c 'python manage.py test unittests -v2'`
 {{% /alert %}}
 
 ### Endpoint validation

dojo/jira_link/views.py

@@ -25,7 +25,7 @@
 logger = logging.getLogger(__name__)
 
 
-# for examples of incoming json, see the unit tests for the webhook: https://github.com/DefectDojo/django-DefectDojo/blob/master/dojo/unittests/test_jira_webhook.py
+# for examples of incoming json, see the unit tests for the webhook: https://github.com/DefectDojo/django-DefectDojo/blob/master/unittests/test_jira_webhook.py
 # or the officials docs (which are not always clear): https://developer.atlassian.com/server/jira/platform/webhooks/
 @csrf_exempt
 @require_POST