Docker volume for media files

DOCKER.md

@@ -1,16 +1,16 @@
 # Running with Docker Compose
 
-The docker-compose.yml in this repo is not intended for production use without first customizing it to fit your specific situation.  Please consider the docker-compose.yml files are templates to create on that fits your needs.
-Docker Compose is acceptable if you want to deploy a containerized DefectDojo to a production environment.
-It is one of the supported [Default installation](setup/README.md) methods.
+The docker-compose.yml file in this repository is fully functional to evaluate DefectDojo in your local environment. 
+
+Although Docker Compose is one of the supported installation methods to deploy a containerized DefectDojo in a production environment, the docker-compose.yml file is not intended for production use without first customizing it to your particular situation. [Running in Production](docs/content/running/running-in-production.md) gives advice on which adjustments are useful for performance and operational reliability. 
+
 
 # Prerequisites
 *  Docker version
     *  Installing with docker-compose requires at least docker 18.09.4 and docker-compose 1.24.0. See "Checking Docker versions" below for version errors during running docker-compose.
 *  Proxies
     *  If you're behind a corporate proxy check https://docs.docker.com/network/proxy/ . 
-*  Known issues
-    * finding images only work in `dev` and `ptvsd` mode. Making them work in `release` mode requires modifications to the docker-compose configuration.
+
 
 # Setup via Docker Compose - introduction
 

Dockerfile.django

@@ -140,4 +140,5 @@ ENV \
   DD_DJANGO_ADMIN_ENABLED="True" \
   DD_TRACK_MIGRATIONS="True" \
   DD_DJANGO_METRICS_ENABLED="False"
+RUN mkdir -p media && mkdir -p media/threat && chown -R ${uid} media
 ENTRYPOINT ["/entrypoint-uwsgi.sh"]

docker-compose.override.dev.yml

@@ -5,9 +5,6 @@ services:
     entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/entrypoint-uwsgi-dev.sh']
     volumes:
       - '.:/app:z'
-      - './media:/app/media/finding_images'
-      - './media/CACHE:/app/media/CACHE'
-      - './media/threat:/app/media/threat'
     environment:
       DD_DEBUG: 'True'
       DD_ADMIN_USER: ${DD_ADMIN_USER:-admin}
@@ -27,8 +24,6 @@ services:
   nginx:
     volumes:
       - './dojo/static/dojo:/usr/share/nginx/html/static/dojo'
-      - './media:/usr/share/nginx/html/media'
-      - './media/CACHE:/usr/share/nginx/html/media/CACHE'
   mysql:
     ports:
       - target: 3306

docker-compose.override.integration_tests.yml

@@ -12,15 +12,20 @@ services:
     entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/app/docker/entrypoint-integration-tests.sh']
     volumes:
       - '.:/app:z'
+      - defectdojo_media_unittest:${DD_MEDIA_ROOT:-/app/media}
     environment:
       DD_BASE_URL: 'http://nginx:8080/'
       DD_ADMIN_USER: ${DD_ADMIN_USER:-admin}
       DD_ADMIN_PASSWORD: ${DD_ADMIN_PASSWORD:-AdminsLoveIntegrationtests!}
       DD_SECRET_KEY: ${DD_SECRET_KEY:-.}
+  nginx:
+    volumes:
+      - defectdojo_media_unittest:/usr/share/nginx/html/media
   uwsgi:
     entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/entrypoint-uwsgi-dev.sh']
     volumes:
       - '.:/app:z'
+      - defectdojo_media_unittest:${DD_MEDIA_ROOT:-/app/media}
     environment:
       DD_DEBUG: 'True'
       DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-mysql://defectdojo:defectdojo@mysql:3306/test_defectdojo}
@@ -49,3 +54,4 @@ services:
        - defectdojo_data_unittest:/var/lib/mysql
 volumes:
   defectdojo_data_unittest: {}
+  defectdojo_media_unittest: {}

docker-compose.override.ptvsd.yml

@@ -5,9 +5,6 @@ services:
     entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/entrypoint-uwsgi-ptvsd.sh']
     volumes:
       - '.:/app:z'
-      - './media:/app/media/finding_images'
-      - './media/CACHE:/app/media/CACHE'
-      - './media/threat:/app/media/threat'
     environment:
       DD_DEBUG: 'True'
     ports:
@@ -21,8 +18,6 @@ services:
   nginx:
     volumes:
       - './dojo/static/dojo:/usr/share/nginx/html/static/dojo'
-      - './media:/usr/share/nginx/html/media'
-      - './media/CACHE:/usr/share/nginx/html/media/CACHE'      
   mysql:
     ports:
       - target: 3306

docker-compose.override.unit_tests.yml

@@ -4,10 +4,13 @@ services:
   nginx:
     image: busybox:1.33.0-musl
     entrypoint: ['echo', 'skipping', 'nginx']
+    volumes:
+      - defectdojo_media_unittest:/usr/share/nginx/html/media
   uwsgi:
     entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/app/docker/entrypoint-unit-tests-devDocker.sh']
     volumes:
       - '.:/app:z'
+      - defectdojo_media_unittest:${DD_MEDIA_ROOT:-/app/media}
     environment:
       DD_DEBUG: 'True'
       DD_TEST_DATABASE_NAME: ${DD_TEST_DATABASE_NAME:-test_defectdojo}
@@ -34,3 +37,4 @@ services:
        - defectdojo_data_unittest:/var/lib/mysql
 volumes:
   defectdojo_data_unittest: {}
+  defectdojo_media_unittest: {}

docker-compose.override.unit_tests_cicd.yml

@@ -4,10 +4,13 @@ services:
   nginx:
     image: busybox:1.33.0-musl
     entrypoint: ['echo', 'skipping', 'nginx']
+    volumes:
+      - defectdojo_media_unittest:/usr/share/nginx/html/media
   uwsgi:
     entrypoint: ['/wait-for-it.sh', 'mysql:3306', '-t', '30', '--', '/app/docker/entrypoint-unit-tests.sh']
     volumes:
       - '.:/app:z'
+      - defectdojo_media_unittest:${DD_MEDIA_ROOT:-/app/media}
     environment:
       DD_DEBUG: 'True'
       DD_TEST_DATABASE_NAME: ${DD_TEST_DATABASE_NAME:-test_defectdojo}
@@ -33,3 +36,4 @@ services:
        - defectdojo_data_unittest:/var/lib/mysql
 volumes:
   defectdojo_data_unittest: {}
+  defectdojo_media_unittest: {}

docker-compose.yml

@@ -1,3 +1,10 @@
+# This docker-compose.yml file  is fully functional to evaluate DefectDojo
+# in your local environment. 
+#
+# Although Docker Compose is one of the supported installation methods to
+# deploy a containerized DefectDojo in a production environment, the 
+# docker-compose.yml file is not intended for production use without first
+# customizing it to your particular situation.
 ---
 version: '3.7'
 services:
@@ -10,6 +17,8 @@ services:
       - uwsgi
     environment:
       NGINX_METRICS_ENABLED: ${NGINX_METRICS_ENABLED:-false}
+    volumes:
+      - defectdojo_media:/usr/share/nginx/html/media
     ports:
       - target: 8080
         published: ${DD_PORT:-8080}
@@ -40,6 +49,7 @@ services:
         - type: bind
           source: ./docker/extra_settings
           target: /app/docker/extra_settings
+        - defectdojo_media:${DD_MEDIA_ROOT:-/app/media}
   celerybeat:
     image: defectdojo/defectdojo-django:latest
     depends_on:
@@ -105,3 +115,4 @@ services:
     image: rabbitmq:3.8.12@sha256:bbd86b54a03a437e67d160462a12dd949746c3f1cdd82add64aa258c292fca29
 volumes:
   defectdojo_data: {}
+  defectdojo_media: {}

docs/content/running/running-in-production.md

@@ -6,16 +6,24 @@ draft: false
 
 
 
-Improving your docker-compose performance
------------------------------------------
+Production with docker-compose
+------------------------------
 
-### Database
+The docker-compose.yml file in this repository is fully functional to evaluate DefectDojo in your local environment. 
 
-Run your database elsewhere. Tweak your docker-compose configuration to
-that effect. If you don\'t, you cannot pretend to be running in
-production.
+Although Docker Compose is one of the supported installation methods to deploy a containerized DefectDojo in a production environment, the docker-compose.yml file is not intended for production use without first customizing it to your particular situation.
 
-### Instance size
+## Database performance and backup
+
+It is recommended to use a dedicated database server and not the preconfigured MySQL database. This will improve the performance of DefectDojo 
+
+In both case, if you use a dedicated database server or if you should decide to use the preconfigured MySQL database, make sure to make regular backups of the data. For a dedicated database server follow the instructions that come with the database server. For the preconfigured MySQL you can use mysqldump, e.g. as described in [How to backup a Docker MySQL database](https://dev.to/grant_bartlett/how-to-backup-a-docker-mysql-database-3nd8).
+
+## Backup of Media files
+
+Media files for uploaded files, including threat models and risk acceptance, are stored in a docker volume. This volume needs to be backed up regularly.
+
+## Instance size
 
 {{% notice note %}}
 Please read the paragraphs below about key processes tweaks.
@@ -32,7 +40,7 @@ is:
     a different disk than your OS\'s for potential performance
     improvements.
 
-### Key processes
+## Key processes
 
 Per <https://github.com/DefectDojo/django-DefectDojo/pull/2813>, it is
 now easy to somewhat improve the uWSGI and celery worker performance.
@@ -79,6 +87,7 @@ You can execute the following command to see the configuration:
 `docker-compose exec celerybeat bash -c "celery -A dojo inspect stats"`
 and see what is in effect.
 
+
 Production with setup.bash
 --------------------------
 

tests/base_test_class.py

@@ -283,18 +283,14 @@ def assertNoConsoleErrors(self):
 
         for entry in WebdriverOnlyNewLogFacade(self.driver).get_log('browser'):
             """
-            images are not working in current docker/travis deployment, so ignore those 404s
-            see: https://github.com/DefectDojo/django-DefectDojo/issues/2045
-            examples:
-            http://localhost:8080/static/dojo/img/zoom-in.cur - Failed to load resource: the server responded with a status of 404 (Not Found)
-            http://localhost:8080/media/CACHE/images/finding_images/1bf9c0b1-5ed1-4b4e-9551-bcbfd198b90a/7d8d9af058566b8f2fe6548d96c63237.jpg - Failed to load resource: the server responded with a status of 404 (Not Found)
+            Images are now working after https://github.com/DefectDojo/django-DefectDojo/pull/3954,
+            but http://localhost:8080/static/dojo/img/zoom-in.cur still produces a 404
 
             The addition of the trigger exception is due to the Report Builder tests. All of the moving objects are from javascrip
             Tooltips are attached to each object and operate fine at human speeds. Selenium moves too fast for tooltips to be
             cleaned up, edited, and displayed, so the issue is only present in the test
             """
-            accepted_javascript_messages = r'((zoom\-in\.cur.*)|(images\/finding_images\/.*)||(uploaded_files\/.*))404\ \(Not\ Found\)|Cannot read property \'trigger\' of null'
-            # accepted_javascript_messages = r'((zoom\-in\.cur.*)|(images\/finding_images\/.*))404\ \(Not\ Found\)|(bootstrap\-chosen\.css\.map)'
+            accepted_javascript_messages = r'(zoom\-in\.cur.*)404\ \(Not\ Found\)|Cannot read property \'trigger\' of null'
 
             if (entry['level'] == 'SEVERE'):
                 # print(self.driver.current_url)  # TODO actually this seems to be the previous url