prevent database connections leaks during batch web requests

.ebextensions/01_packages.config

@@ -2,4 +2,4 @@
 packages:
   yum:
     git: []
-    postgresql92-devel: []
+    postgresql-devel: []

.ebextensions/02_python.config

@@ -5,7 +5,7 @@ container_commands:
     # command: chmod g+s /opt/python/log
     command: chmod g+w /opt/python/log/*
   02_change_owner:
-    command: chown root:wsgi /opt/python/log/*
+    command: chown webapp:webapp /opt/python/log/*
   03_migrate:
     command: "python manage.py migrate --noinput"
     leader_only: true
@@ -17,5 +17,5 @@ container_commands:
   05_collectstatic:
     command: "python manage.py collectstatic --noinput"
     leader_only: true
-  06_change_cache_permissions:
-    command: chmod 777 /opt/python/ondeck/app/*_cache*
+#  06_change_cache_permissions:
+#    command: chmod 777 /opt/python/ondeck/app/*_cache*

Makefile

@@ -0,0 +1,42 @@
+DEV_DB_CONTAINER_NAME = crispycrunch-dev
+TEST_DB_CONTAINER_NAME = crispycrunch-test
+
+.PHONY: init
+init:
+	pip install -r requirements.txt
+
+.PHONY: setup-develop
+setup-develop:
+	pip install -e .'[dev]'
+	pre-commit install
+
+.PHONY: pre-commit
+pre-commit:
+	pre-commit run --all-files
+
+.PHONY: lint
+lint:
+	flake8 . --count --statistics --exit-zero
+	python -m pylint ./opencell
+
+.PHONY: test
+test:
+	pytest -v --ignore ./client/node_modules
+
+.PHONY: start-dev-db
+start-dev-db: drop-dev-db
+	docker create \
+	--name $(DEV_DB_CONTAINER_NAME) \
+	-e POSTGRES_USER=crispycrunch \
+	-e POSTGRES_PASSWORD=password \
+	-e POSTGRES_DB=crispycrunchdb \
+	-p 5435:5432 \
+	postgres
+	docker start $(DEV_DB_CONTAINER_NAME) && sleep 2
+
+.PHONY: drop-dev-db
+drop-dev-db:
+	-docker rm --force $(DEV_DB_CONTAINER_NAME);
+
+start-app:
+	python manage.py runserver

crispycrunch/settings.py

@@ -210,10 +210,11 @@ def process_exception(self, request, exception):
     DATABASES = {
         'default': {
             'ENGINE': 'django.db.backends.postgresql',
-            'NAME': 'postgres',
-            'USER': 'postgres',
+            'NAME': 'crispycrunchdb',
+            'USER': 'crispycrunch',
+            'PASSWORD': 'password',
             'HOST': 'localhost',
-            'PORT': '5432',
+            'PORT': '5435',
         }
     }
 

main/templates/home.html

@@ -5,11 +5,7 @@
 <h2>Get Started</h2>
 
 <ul>
-  <li class="h5">
-    View <a target="_blank" href="https://crispycrunch.czbiohub.org/main/primer-selection/91/experiment-summary/?url_auth_token=AAAAD6DADtu-27EV2aBL2x6BrSY%3AsYYjdE_ZJHYKAFvy7XjberSIR1c">demo experiment</a>
-    and <a target="_blank" href="https://crispycrunch.czbiohub.org/main/analysis/11/results/?url_auth_token=AAAAD6DADtu-27EV2aBL2x6BrSY%3AsYYjdE_ZJHYKAFvy7XjberSIR1c">demo analysis</a>
-  </li>
-  <li class="h5">Read <a href="/howto/">how to guide</a></li>
+  <li class="h5">Read the <a href="/howto/">how to guide</a></li>
   <li class="h5">Create an <a href="/main/signup/">account</a></li>
   <li class="h5">Design a <a href="/main/experiment/">new experiment</a></li>
   <li class="h5">Start a <a href="/main/analysis/">new analysis</a></li>

webscraperequest/batchrequest.py

@@ -8,6 +8,7 @@
 from unittest import mock  # noqa
 
 from django.db import models
+from django.db import connection
 
 try:
     from .scraperequest import *
@@ -18,6 +19,24 @@
 logger = logging.getLogger(__name__)
 
 
+def _request_callback(future, model_instance, field_name, requester, index=None) -> None:
+    try:
+        result = future.result()
+        result['success'] = result.get('success', True)
+        logger.debug('{} result inserted into database'.format(requester.__class__))
+    except Exception as e:
+        result['success'] = result.get('success', False)
+        result['error'] = getattr(e, 'message', str(e))
+        logger.error('Error inserting into index {}: {}'.format(index, str(e)))
+    finally:
+        result['end_time'] = int(time.time())
+        model_instance.__dict__[str(field_name)][index].update(result)
+        model_instance.save()
+
+        # explicitly close the connection inside the thread to prevent leaked connections
+        connection.close()
+
+
 class BaseBatchWebRequest:
     """
     Manages a parallel batch of web requests. Intermediate and final results are
@@ -48,8 +67,14 @@ def start(self, largs: List[list], keys: List[int] = []) -> None:
         pool = ThreadPoolExecutor(self.max_workers)
         for i, args in enumerate(largs):
             logger.debug('{} submitted to thread pool'.format(self.requester.__class__))
-            pool.submit(self._request, args).add_done_callback(
-                functools.partial(self._insert, index=i))
+            callback = functools.partial(
+                _request_callback,
+                model_instance=self.model_instance,
+                field_name=self.field_name,
+                requester=self.requester,
+                index=i
+            )
+            pool.submit(self._request, args).add_done_callback(callback)
         pool.shutdown(wait=False)
 
     def get_batch_status(self) -> 'BatchStatus':  # forward ref for typing
@@ -97,22 +122,6 @@ def _request(self, args: list) -> Dict[str, Any]:
                 'error': getattr(e, 'message', str(e)),
             }
 
-    def _insert(self, future, index=None) -> None:
-        try:
-            result = future.result()
-            result['success'] = result.get('success', True)
-            logger.debug('{} result inserted into database'.format(self.requester.__class__))
-        except Exception as e:
-            logger.error('Error inserting into index {}: {}'
-                         .format(index, str(e)))
-            result['success'] = result.get('success', False)
-            result['error'] = getattr(e, 'message', str(e))
-        finally:
-            result['end_time'] = int(time.time())
-            self.model_instance.__dict__[str(self.field_name)][index].update(result)
-            self.model_instance.save()
-
-
 class BatchStatus:
 
     def __init__(