Features/elasticsearch (#30)

* WIP

WIP

* WIP

* WIP

* search

docker-compose.yml

@@ -13,14 +13,6 @@ services:
       - redis
     volumes:
       - .:/kerckhoff
-#      - node_modules:/kerckhoff/node_modules
-  # svc:
-  #   image: kerckhoff
-  #   command: npm run watch
-  #   restart: always
-  #   volumes:
-  #     - .:/kerckhoff
-  #     - node_modules:/kerckhoff/node_modules
   db:
     image: postgres:latest
     ports:
@@ -29,6 +21,12 @@ services:
     image: redis:alpine
     ports:
       - "6379:6379"
+  es:
+    image: blacktop/elasticsearch
+    ports:
+      - "9200:9200"
+    volumes:
+      - es_data:/usr/share/elasticsearch/data
 
 volumes:
-  node_modules:
+  es_data:

jsconfig.json

@@ -1,5 +1,9 @@
 {
     "include": [
       "./assets/**/*"
+    ],
+    "exclude": [
+      "node_modules",
+      "./assets/bundles/**/*"
     ]
   }

kerckhoff/__init__.py

@@ -0,0 +1,3 @@
+from .es import init_es
+
+es = init_es()

kerckhoff/es.py

@@ -0,0 +1,11 @@
+import logging
+from django.conf import settings
+from elasticsearch import Elasticsearch
+from elasticsearch_dsl.connections import connections
+
+logger = logging.getLogger(settings.APP_NAME)
+
+def init_es() -> Elasticsearch:
+    logger.info("Connecting to elasticsearch...")
+    # this creates the global connection pool to elasticsearch
+    return connections.create_connection(hosts=[settings.ES_HOST,])

kerckhoff/exceptions.py

@@ -0,0 +1,12 @@
+class KerckhoffCustomException(Exception):
+    status_code = None
+    error_message = None
+    is_an_error_response = True
+    def __init__(self, error_message):
+        Exception.__init__(self)
+        self.error_message = error_message
+    def to_dict(self):
+        return {'message': self.error_message}
+
+class UserError(KerckhoffCustomException):
+    status_code = 400

kerckhoff/middleware.py

@@ -0,0 +1,27 @@
+from django.http import JsonResponse
+from django.conf import settings
+from django.utils.deprecation import MiddlewareMixin
+import logging
+import traceback
+
+logger = logging.getLogger(settings.APP_NAME)
+
+def is_registered(exception: Exception):
+    try:
+        return exception.is_an_error_response
+    except AttributeError:
+        return False
+
+class RequestExceptionMiddleware(MiddlewareMixin):
+    def process_exception(self, request, exception):
+        if is_registered(exception):
+            status = exception.status_code
+            exception_dict = exception.to_dict()
+        else:
+            status = 500
+            exception_dict = {'message': 'Unexpected server error'}
+            logger.error("ERROR 500 - %s" %  traceback.format_exc())
+            if settings.DEBUG:
+                raise exception
+
+        return JsonResponse(exception_dict, status=status)

kerckhoff/settings.py

@@ -12,6 +12,8 @@
 
 import os
 import environ
+import logging.config
+from django.utils.log import DEFAULT_LOGGING
 
 root = environ.Path(__file__) - 3 # three folder back (/a/b/c/ - 3 = /)
 env = environ.Env(DEBUG=(bool, False),) # set default values and casting
@@ -20,7 +22,7 @@
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-
+APP_NAME = "kerckhoff"
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
 
@@ -57,7 +59,8 @@
 
     'pages',
     'user_profile',
-    'packages'
+    'packages',
+    'search'
 ]
 
 MIDDLEWARE = [
@@ -70,6 +73,7 @@
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'kerckhoff.middleware.RequestExceptionMiddleware',
 ]
 
 ROOT_URLCONF = 'kerckhoff.urls'
@@ -92,6 +96,58 @@
     },
 ]
 
+# Disable Django's logging setup
+LOGGING_CONFIG = None
+
+LOGLEVEL = os.environ.get('LOGLEVEL', 'info').upper()
+
+logging.config.dictConfig({
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'default': {
+            # exact format is not important, this is the minimum information
+            'format': '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
+        },
+        'django.server': DEFAULT_LOGGING['formatters']['django.server'],
+    },
+    'handlers': {
+        # console logs to stderr
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'default',
+        },
+        # Add Handler for Sentry for `warning` and above
+        # 'sentry': {
+        #     'level': 'WARNING',
+        #     'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
+        # },
+        'django.server': DEFAULT_LOGGING['handlers']['django.server'],
+    },
+    'loggers': {
+        # default for all undefined Python modules
+        '': {
+            'level': 'WARNING',
+            'handlers': ['console',],
+        },
+        # Our application code
+        'kerckhoff': {
+            'level': LOGLEVEL,
+            'handlers': ['console',],
+            # Avoid double logging because of root logger
+            'propagate': False,
+        },
+        # # Prevent noisy modules from logging to Sentry
+        # 'noisy_module': {
+        #     'level': 'ERROR',
+        #     'handlers': ['console',],
+        #     'propagate': False,
+        # },
+        # Default runserver request logging
+        'django.server': DEFAULT_LOGGING['loggers']['django.server'],
+    },
+})
+
 WSGI_APPLICATION = 'kerckhoff.wsgi.application'
 
 
@@ -185,6 +241,8 @@
     }
 }
 
+# Elasticsearch
+ES_HOST = env('ES_HOST', default="es")
 
 # CORS