Merge pull request #380 from WHOIGit/new_build

new build and deployment process

.dockerignore

@@ -6,3 +6,6 @@ docker-compose.yml
 pg_data
 ifcb_data
 nginx
+.gitignore
+.git
+**/.git

.gitignore

@@ -53,7 +53,6 @@ coverage.xml
 
 # Django stuff:
 *.log
-local_settings.py
 db.sqlite3
 
 # Flask stuff:
@@ -116,6 +115,7 @@ pyifcb
 pg_data
 ifcb_data
 static
+local_settings.py
 
 .DS_Store
 .vscode

Dockerfile

@@ -1,38 +1,23 @@
-from continuumio/miniconda3
+FROM python:3.11-slim
 
 # geospatial libraries
-RUN apt-get update && apt-get install -y binutils libproj-dev
+RUN apt-get update && \
+    apt-get install -y binutils git libproj-dev libgdal-dev libpoppler-dev && \
+    rm -rf /var/lib/apt/lists/*
 
-RUN conda config --remove channels defaults
-RUN conda config --append channels conda-forge
+WORKDIR /build
+COPY requirements.txt .
 
-RUN conda update conda
-
-# nomkl to reduce image size (mkl is large)
-RUN conda install -c conda-forge nomkl conda-merge mamba
-
-# install pyifcb and ifcbdb dependencies first
-# pyifcb must be cloned into the same directory as this dockerfile
-
-WORKDIR /pyifcb
-COPY ./pyifcb .
-COPY ./pyifcb/environment.yml /envs/pyifcb_env.yml
+RUN pip install --no-cache-dir -r requirements.txt
 
 WORKDIR /ifcbdb
-COPY environment.yml /envs/ifcbdb_env.yml
-
-WORKDIR /envs
-RUN conda-merge pyifcb_env.yml ifcbdb_env.yml > merged-environment.yml
-RUN cat merged-environment.yml
+COPY ./ifcbdb .
 
-RUN mamba env update -n root -f merged-environment.yml
+WORKDIR /utilities
+COPY ./utilities .
 
-WORKDIR /pyifcb
-RUN python setup.py develop
+WORKDIR /ifcbdb
 
 EXPOSE 8000
 
-# descend into app directory
-WORKDIR /ifcbdb
-
 CMD gunicorn --bind :8000 ifcbdb.wsgi:application --reload

README.md

@@ -1,2 +1,67 @@
-# ifcbdb
-IFCB dashboard
+# IFCB Dashboard
+
+The IFCB dashboard provides a web interface for interacting with IFCB data, metadata, and imagery.
+Users can load raw data and data products such as classification results into the dashboard, where they can be viewed.
+In addition, metadata can be uploaded to the dashboard to allow for geospatial referencing / mapping.
+
+## Installation
+
+The dashboard is deployed using Docker. Docker will need to be installed before you can run the dashboard.
+
+To configure the dashboard, copy `dotenv.template` to `.env` and edit the `.env` file with configuration parameters.
+
+Here are the key parameters to edit in that file:
+
+* `PRIMARY_DATA_DIR` refers to where your IFCB data is located on your system.
+* `POSTGIS_IMAGE` needs to be configured if you are using Apple Silicon or another ARM-based system; otherwise the default value will work.
+* `NGINX_TEMPLATE` allows you to specify an alternative NGINX configuration. Using the default is almost always preferable, as it is suitable for most deployment scenarios.
+* `HOST` should be the fully qualified domain name of the computer where you are running the dashboard. The default is `localhost` which is used for testing purposes only.
+* `HTTP_PORT` and `HTTPS_PORT` control which ports the dashboard will respond at. The defaults are 80 and 443 and they should work unless you are already running a web service on your computer that is already listening on those ports.
+
+### Security configuration
+
+To run the dashboard, you'll need an SSL certificate (unless you're handling TLS termination in a web proxy, see below). This should be able to be provided by your host organization. If you cannot acquire an SSL certificate, you will need to generate a "self-signed" certificate but that configuration should only be used for testing, because it will give users a security warning in their browsers that strongly encourage them to reject access to the site.
+
+Once you have acquired the certificate and placed the certificate file and key file in appropriate locations on your system, you will need to configure the dashboard to access that certificate using the `SSL_CERT` and `SSL_KEY` parameters in `.env`, using the path to each file.
+
+In addition to SSL, there is a security parameter in the `.env` file called `DJANGO_SECRET_KEY`. You will need to change that parameter and set it to some unique value that can't be easily guessed.
+
+If you're terminating TLS at the web proxy layer you can run ifcbdb in HTTP-only mode by setting `NGINX_TEMPLATE=./nginx.conf.template`.
+
+### Building the image yourself
+
+If you don't want to use the image from Docker Hub (for instance, if you've made modifications) you can build it yourself. Once you build and tag the image from the provided Dockerfile, configure the `IFCBDB_IMAGE` parameter in `.env` to refer to your image tag.
+
+### Notes on ARM platforms (e.g., Apple Silicon)
+
+Note that as of February 2024 the official postgis image in Docker Hub is currently not available for ARM platforms. An alternate image is provided as an example configuration in `dotenv.template`.
+
+## Running the dashboard
+
+Once you have configured `.env`, you can bring the dashboard up by running `docker-compose up -d` from the directory containing the `.env` and `docker-compose.yml` files.
+
+### Post-installation steps
+
+After starting the dashboard for the first time, you'll need to run the following commands to initialize the dashboard's backend database and files.
+
+```
+docker compose exec ifcbdb python manage.py migrate
+docker compose exec ifcbdb python manage.py collectstatic
+```
+
+### Logging in for the first time
+
+You will need to create a "superuser" account, specifying its username and password. To do that, run this command to create your user and password.
+
+```
+docker compose exec ifcbdb python manage.py createsuperuser
+```
+
+If you need to create the superuser non-interactively, you can set the `DJANGO_SUPERUSER_PASSWORD` environment variable
+([see Django docs](https://docs.djangoproject.com/en/5.0/ref/django-admin/#envvar-DJANGO_SUPERUSER_PASSWORD)).
+
+### Advanced configuration
+
+If you need to set configuration options beyond the available environment variables, you can create a
+`local_settings.py` file and set environment variable `LOCAL_SETTINGS` to its path. The file will
+be imported at the end of Django's `settings.py`, allowing you to override any previous setting.

bin/assets.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-docker exec -it ifcbdb python manage.py collectstatic --no-input
+docker compose exec ifcbdb python manage.py collectstatic --no-input

bin/clearcache.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-docker exec -it ifcbdb python manage.py clearcache
+docker compose exec ifcbdb python manage.py clearcache

bin/migrate.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-docker exec -it ifcbdb python manage.py migrate
+docker compose exec ifcbdb python manage.py migrate

bin/pgdump.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-docker exec -it postgres pg_dump -U ifcb ifcb
+docker compose exec postgres pg_dump -U ifcb ifcb

bin/pshell.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-docker exec -it ifcbdb python manage.py shell
+docker compose exec python manage.py shell
 

bin/psql.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-docker exec -it postgres psql -U ifcb -W
+docker compose exec postgres psql -U ifcb -W
 

bin/shell.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-docker exec -it ifcbdb bash
+docker compose exec bash
 

docker-compose.yml

@@ -2,13 +2,18 @@ version: '3'
 
 services:
   ifcbdb:
-    build: .
-    container_name: ifcbdb
+    image: ${IFCBDB_IMAGE:-whoi/ifcb-dashboard:latest}
+    environment:
+      - NGINX_HOST=${HOST:-localhost}
+      - NGINX_HTTP_PORT=${HTTP_PORT:-80}
+      - NGINX_HTTPS_PORT=${HTTPS_PORT:-443}
+      - DEFAULT_DATASET=${DEFAULT_DATASET:-}
+      - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY:-changeme}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-ifcb}
     volumes:
-      - ./pyifcb:/pyifcb
-      - ./ifcbdb:/ifcbdb
-      - ./static:/static
-      - ./ifcb_data:/data
+      - nginx-static:/static
+      - ${PRIMARY_DATA_DIR:-./ifcb_data}:/data
+      - ${LOCAL_SETTINGS:-/dev/null}:/ifcbdb/ifcbdb/local_settings.py
     networks:
       - nginx_network
       - postgres_network
@@ -20,13 +25,14 @@ services:
       - redis
 
   celery:
-    build: .
-    container_name: celery
+    image: ${IFCBDB_IMAGE:-whoi/ifcb-dashboard:latest}
     command: celery -A ifcbdb worker -l info
+    environment:
+      - DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY:-changeme}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-ifcb}
     volumes:
-      - ./pyifcb:/pyifcb
-      - ./ifcbdb:/ifcbdb
-      - ./ifcb_data:/data
+      - ${PRIMARY_DATA_DIR:-./ifcb_data}:/data
+      - ${LOCAL_SETTINGS:-/dev/null}:/ifcbdb/ifcbdb/local_settings.py
     depends_on:
       - postgres
       - memcached
@@ -37,40 +43,43 @@ services:
       - memcached_network
 
   nginx:
-    image: nginx:latest
-    container_name: nginx
+    image: ${NGINX_IMAGE:-nginx:1.25}
+    environment:
+      - NGINX_HOST=${HOST:-localhost}
+      - NGINX_HTTP_PORT=${HTTP_PORT:-80}
+      - NGINX_HTTPS_PORT=${HTTPS_PORT:-443}
     ports:
-      - 8000:80
+      - ${HTTP_PORT:-80}:80
+      - ${HTTPS_PORT:-443}:443
     volumes:
-      - ./nginx:/etc/nginx/conf.d
-      - ./static:/static
+      - ${NGINX_TEMPLATE:-./nginx-ssl.conf.template}:/etc/nginx/templates/default.conf.template
+      - nginx-static:/static
+      - ${SSL_KEY}:/ssl/ssl.key:ro
+      - ${SSL_CERT}:/ssl/ssl.cer:ro
     depends_on:
       - ifcbdb
     networks:
       - nginx_network
 
   postgres:
-    image: mdillon/postgis:11-alpine
-    container_name: postgres
+    image: ${POSTGIS_IMAGE:-postgis/postgis:16-3.4}
     environment:
       - POSTGRES_USER=ifcb
-      - POSTGRES_PASSWORD=ifcb
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-ifcb}
       - POSTGRES_DB=ifcb
     volumes:
-      - ./pg_data:/var/lib/postgresql/data
+      - postgis-data:/var/lib/postgresql/data
     networks:
       - postgres_network
 
   memcached:
-    image: memcached:latest
-    container_name: memcached
+    image: ${MEMCACHED_IMAGE:-memcached:1.6}
     command: ["-m", "64m"]
     networks:
       - memcached_network
 
   redis:
-    image: redis:latest
-    container_name: redis
+    image: ${REDIS_IMAGE:-redis:7.2}
     networks:
       - redis_network
 
@@ -83,3 +92,7 @@ networks:
     driver: bridge
   redis_network:
     driver: bridge
+
+volumes:
+  postgis-data:
+  nginx-static: