diff --git a/.ebextensions/01_packages.config b/.ebextensions/01_packages.config
index f99e366..b214ed7 100644
--- a/.ebextensions/01_packages.config
+++ b/.ebextensions/01_packages.config
@@ -2,4 +2,4 @@
packages:
yum:
git: []
- postgresql92-devel: []
\ No newline at end of file
+ postgresql-devel: []
\ No newline at end of file
diff --git a/.ebextensions/02_python.config b/.ebextensions/02_python.config
index 1fa8c9b..09b2345 100644
--- a/.ebextensions/02_python.config
+++ b/.ebextensions/02_python.config
@@ -5,7 +5,7 @@ container_commands:
# command: chmod g+s /opt/python/log
command: chmod g+w /opt/python/log/*
02_change_owner:
- command: chown root:wsgi /opt/python/log/*
+ command: chown webapp:webapp /opt/python/log/*
03_migrate:
command: "python manage.py migrate --noinput"
leader_only: true
@@ -17,5 +17,5 @@ container_commands:
05_collectstatic:
command: "python manage.py collectstatic --noinput"
leader_only: true
- 06_change_cache_permissions:
- command: chmod 777 /opt/python/ondeck/app/*_cache*
\ No newline at end of file
+# 06_change_cache_permissions:
+# command: chmod 777 /opt/python/ondeck/app/*_cache*
\ No newline at end of file
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..e155ea1
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,42 @@
+DEV_DB_CONTAINER_NAME = crispycrunch-dev
+TEST_DB_CONTAINER_NAME = crispycrunch-test
+
+.PHONY: init
+init:
+ pip install -r requirements.txt
+
+.PHONY: setup-develop
+setup-develop:
+ pip install -e .'[dev]'
+ pre-commit install
+
+.PHONY: pre-commit
+pre-commit:
+ pre-commit run --all-files
+
+.PHONY: lint
+lint:
+ flake8 . --count --statistics --exit-zero
+ python -m pylint ./opencell
+
+.PHONY: test
+test:
+ pytest -v --ignore ./client/node_modules
+
+.PHONY: start-dev-db
+start-dev-db: drop-dev-db
+ docker create \
+ --name $(DEV_DB_CONTAINER_NAME) \
+ -e POSTGRES_USER=crispycrunch \
+ -e POSTGRES_PASSWORD=password \
+ -e POSTGRES_DB=crispycrunchdb \
+ -p 5435:5432 \
+ postgres
+ docker start $(DEV_DB_CONTAINER_NAME) && sleep 2
+
+.PHONY: drop-dev-db
+drop-dev-db:
+ -docker rm --force $(DEV_DB_CONTAINER_NAME);
+
+start-app:
+ python manage.py runserver
\ No newline at end of file
diff --git a/crispycrunch/settings.py b/crispycrunch/settings.py
index 20ee12b..227b076 100644
--- a/crispycrunch/settings.py
+++ b/crispycrunch/settings.py
@@ -210,10 +210,11 @@ def process_exception(self, request, exception):
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
- 'NAME': 'postgres',
- 'USER': 'postgres',
+ 'NAME': 'crispycrunchdb',
+ 'USER': 'crispycrunch',
+ 'PASSWORD': 'password',
'HOST': 'localhost',
- 'PORT': '5432',
+ 'PORT': '5435',
}
}
diff --git a/main/templates/home.html b/main/templates/home.html
index 97f6390..d8cc170 100644
--- a/main/templates/home.html
+++ b/main/templates/home.html
@@ -5,11 +5,7 @@
Get Started
- -
- View demo experiment
- and demo analysis
-
- - Read how to guide
+ - Read the how to guide
- Create an account
- Design a new experiment
- Start a new analysis
diff --git a/webscraperequest/batchrequest.py b/webscraperequest/batchrequest.py
index e0a1b1e..23e739c 100644
--- a/webscraperequest/batchrequest.py
+++ b/webscraperequest/batchrequest.py
@@ -8,6 +8,7 @@
from unittest import mock # noqa
from django.db import models
+from django.db import connection
try:
from .scraperequest import *
@@ -18,6 +19,24 @@
logger = logging.getLogger(__name__)
+def _request_callback(future, model_instance, field_name, requester, index=None) -> None:
+ try:
+ result = future.result()
+ result['success'] = result.get('success', True)
+ logger.debug('{} result inserted into database'.format(requester.__class__))
+ except Exception as e:
+ result['success'] = result.get('success', False)
+ result['error'] = getattr(e, 'message', str(e))
+ logger.error('Error inserting into index {}: {}'.format(index, str(e)))
+ finally:
+ result['end_time'] = int(time.time())
+ model_instance.__dict__[str(field_name)][index].update(result)
+ model_instance.save()
+
+ # explicitly close the connection inside the thread to prevent leaked connections
+ connection.close()
+
+
class BaseBatchWebRequest:
"""
Manages a parallel batch of web requests. Intermediate and final results are
@@ -48,8 +67,14 @@ def start(self, largs: List[list], keys: List[int] = []) -> None:
pool = ThreadPoolExecutor(self.max_workers)
for i, args in enumerate(largs):
logger.debug('{} submitted to thread pool'.format(self.requester.__class__))
- pool.submit(self._request, args).add_done_callback(
- functools.partial(self._insert, index=i))
+ callback = functools.partial(
+ _request_callback,
+ model_instance=self.model_instance,
+ field_name=self.field_name,
+ requester=self.requester,
+ index=i
+ )
+ pool.submit(self._request, args).add_done_callback(callback)
pool.shutdown(wait=False)
def get_batch_status(self) -> 'BatchStatus': # forward ref for typing
@@ -97,22 +122,6 @@ def _request(self, args: list) -> Dict[str, Any]:
'error': getattr(e, 'message', str(e)),
}
- def _insert(self, future, index=None) -> None:
- try:
- result = future.result()
- result['success'] = result.get('success', True)
- logger.debug('{} result inserted into database'.format(self.requester.__class__))
- except Exception as e:
- logger.error('Error inserting into index {}: {}'
- .format(index, str(e)))
- result['success'] = result.get('success', False)
- result['error'] = getattr(e, 'message', str(e))
- finally:
- result['end_time'] = int(time.time())
- self.model_instance.__dict__[str(self.field_name)][index].update(result)
- self.model_instance.save()
-
-
class BatchStatus:
def __init__(