feat: Use postgres with nbgrader (#339)

Makefile

@@ -30,6 +30,7 @@ venv:
 	test -d $(VENV_NAME) || virtualenv -p python3 $(VENV_NAME)
 	${PYTHON} -m pip install --upgrade pip
 	${PYTHON} -m pip install -r requirements.txt
+	${VENV_BIN}/ansible-galaxy collection install community.general --ignore-certs 
 
 deploy: prepare
 	${VENV_BIN}/ansible-playbook -i ansible/hosts \

README.md

@@ -114,7 +114,7 @@ Click on the `Grader Console` tab and follow the steps available within the nbgr
 
 * **Data Directories**: This repo uses `docker-compose` to start all services and data volumes for JupyterHub, notebook directories, databases, and the `nbgrader exchange` directory using mounts from the host's file system.
 
-* **Databases**: This setup relies on a standard `postgres` database running in its own container for the JupyterHub application and another separate and optional Postgres database for lab environments (useful to connect from user notebooks).
+* **Databases**: This setup replaces the default SQLite databases with standard Postgres databases running in their own containers. (You may use Postgres DB's running in other locations by updating connections strings and configuration values). The databases are used for the JupyterHub application, the Postgres laboratory environments that need access to database(s) for labs, assignments, etc., and a database for the Nbgrader application.
 
 * **Network**: An external bridge network named `jupyter-network` is used by default. The grader service and the user notebooks are attached to this network.
 
@@ -476,6 +476,11 @@ The services included with this setup rely on environment variables to work prop
 | POSTGRES_USER | `string` | Postgres database username | `jupyterhub` |
 | POSTGRES_PASSWORD | `string` | Postgres database password | `jupyterhub` |
 | POSTGRES_HOST | `string` | Postgres host | `jupyterhub-db` |
+| POSTGRES_NBGRADER_DB | `string` | Postgres database name for nbgrader| `nbgrader` |
+| POSTGRES_NBGRADER_HOST | `string` | Postgres host for Nbgrader | `postgres-nbgrader`  |
+| POSTGRES_NBGRADER_PASSWORD | `string` | Postgres password for Nbgrader | `nbgrader` |
+| POSTGRES_NBGRADER_PORT | `string` | Postgres port for Nbgrader | `5432` |
+| POSTGRES_NBGRADER_USER | `string` | Postgres username for Nbgrader | `nbgrader` |
 | SHARED_FOLDER_ENABLED | `string` | Specifies the use of shared folder (between grader and student notebooks)  | `True` |
 | SPAWNER_MEM_LIMIT | `string` | Spawner memory limit | `2G` |
 | SPAWNER_CPU_LIMIT | `string` | Spawner cpu limit | `0.5` |
@@ -494,6 +499,11 @@ The services included with this setup rely on environment variables to work prop
 | MNT_ROOT | `string` | Notebook grader user id | `/mnt` |
 | NB_UID | `string` | Notebook grader user id | `10001` |
 | NB_GID | `string` | Notebook grader user id | `100` |
+| POSTGRES_NBGRADER_DB | `string` | Postgres database name for nbgrader| `nbgrader` |
+| POSTGRES_NBGRADER_HOST | `string` | Postgres host for Nbgrader | `postgres-nbgrader`  |
+| POSTGRES_NBGRADER_PASSWORD | `string` | Postgres password for Nbgrader | `nbgrader` |
+| POSTGRES_NBGRADER_PORT | `string` | Postgres port for Nbgrader | `5432` |
+| POSTGRES_NBGRADER_USER | `string` | Postgres username for Nbgrader | `nbgrader` |
 | SHARED_FOLDER_ENABLED | `string` | Specifies the use of shared folder (between grader and student notebooks)  | `True` |
 
 ---

ansible/group_vars/all.yml

@@ -27,6 +27,9 @@ docker_illumidesk_grader_image: "{{ docker_illumidesk_grader_image_param | defau
 # setup-course image
 docker_setup_course_image: "{{ docker_setup_course_image_param | default('illumidesk/setup-course:latest') }}"
 
+# postgres image
+docker_postgres_image: "{{ docker_postgres_image_param | default('postgres:11.6-alpine') }}"
+
 # dockerfiles
 docker_jupyterhub_dockerfile: "{{ docker_jupyterhub_dockerfile_param | default('Dockerfile.jhub') }}"
 docker_setup_course_dockerfile: "{{ docker_setup_course_dockerfile_param | default('Dockerfile.setup-course') }}"
@@ -77,3 +80,17 @@ shared_folder_enabled: "{{ shared_folder_enabled_param | default('true') }}"
 # spawner
 spawner_mem_limit: "{{ spawner_mem_limit_param | default('2G') }}"
 spawner_cpu_limit: "{{ spawner_cpu_limit_param | default('0.5') }}"
+
+# Postgres settings for NBgrader
+postgres_nbgrader_host: "{{ postgres_nbgrader_host_param | default('postgres-nbgrader') }}"
+postgres_nbgrader_user: "{{ postgres_nbgrader_user_param | default('postgres') }}"
+postgres_nbgrader_password: "{{ postgres_nbgrader_password_param | default('postgres') }}"
+postgres_nbgrader_port: "{{ postgres_nbgrader_port_param | default('5432') }}"
+postgres_nbgrader_dbname: "{{ postgres_nbgrader_dbname_param | default('nbgrader') }}"
+
+# Postgres settings for JupyterHub
+postgres_jupyterhub_host: "{{ postgres_jupyterhub_host_param | default('jupyterhub-db') }}"
+postgres_jupyterhub_user: "{{ postgres_jupyterhub_user_param | default('illumidesk') }}"
+postgres_jupyterhub_password: "{{ postgres_jupyterhub_password_param | default('illumidesk') }}"
+postgres_jupyterhub_port: "{{ postgres_jupyterhub_port_param | default('5432') }}"
+postgres_jupyterhub_dbname: "{{ postgres_jupyterhub_dbname_param | default('illumidesk') }}"

ansible/hosts.example

@@ -115,4 +115,14 @@ all:
       # spawner_mem_limit: 2G
 
       # uncomment and add your preferred cpu limit settings for user workspaces
-      # spawner_cpu_limit: 0.5
+      # spawner_cpu_limit: 0.5
+
+      ## NBGrader Database settings
+      #-------------------
+      # Uncomment and change the values below as needed. The values below reflect the defaults as
+      # commented in the docs.
+      # postgres_nbgrader_dbname: nbgrader
+      # postgres_nbgrader_host: postgres-nbgrader
+      # postgres_nbgrader_port: nbgrader
+      # postgres_nbgrader_user: nbgrader
+      # postgres_nbgrader_password: nbgrader

ansible/roles/jupyterhub/files/requirements.txt

@@ -2,7 +2,7 @@
 jupyterhub>=1.1.0
 
 # postgres
-psycopg2-binary==2.8.5
+psycopg2-binary==2.8.6
 
 # traefik (reverse proxy)
 jupyterhub-traefik-proxy==0.1.6

ansible/roles/jupyterhub/tasks/main.yml

@@ -7,6 +7,10 @@
   docker_volume:
     name: db-labs-data
 
+- name: create external postgres data volume for nbgrader
+  docker_volume:
+    name: db-nbgrader-data
+
 - name: create external jupyterhub and setup-course data volume
   docker_volume:
     name: jupyterhub-data

ansible/roles/jupyterhub/templates/docker-compose.yml.j2

@@ -23,7 +23,7 @@ services:
       - "traefik.enable=true"
       - "traefik.http.routers.jhub_router.service=jupyterhub"
   jupyterhub-db:
-    image: postgres:11.6-alpine
+    image: {{docker_postgres_image}}
     container_name: jupyterhub-db
     restart: always
     env_file:
@@ -53,9 +53,19 @@ services:
       - {{mnt_root}}:{{mnt_root}}
       - data:/srv/jupyterhub
       - {{ working_dir }}:{{ working_dir }}
+  {{postgres_nbgrader_host}}:
+    image: {{docker_postgres_image}}
+    container_name: {{postgres_nbgrader_host}}
+    restart: always
+    environment:
+      - POSTGRES_DB={{postgres_nbgrader_dbname}}
+      - POSTGRES_USER={{postgres_nbgrader_user}}
+      - POSTGRES_PASSWORD={{postgres_nbgrader_password}}
+    volumes:
+      - db-nbgrader:/var/lib/postgresql/data
 {% if postgres_labs_enabled is sameas true %}
   postgres-labs:
-    image: postgres:11.6-alpine
+    image: {{docker_postgres_image}}
     container_name: postgres-labs
     restart: always
     environment:
@@ -76,6 +86,9 @@ volumes:
   db-labs:
     external:
       name: db-labs-data
+  db-nbgrader:
+    external:
+      name: db-nbgrader-data
 
 networks:
   default:

ansible/roles/jupyterhub/templates/env.jhub.j2

@@ -32,12 +32,19 @@ JUPYTERHUB_ADMIN_USER={{admin_user}}
 ORGANIZATION_NAME={{org_name}}
 
 # postgres
-POSTGRES_DB=illumidesk
-POSTGRES_USER=illumidesk
-POSTGRES_PASSWORD=illumidesk
-POSTGRES_HOST=jupyterhub-db
+POSTGRES_DB={{postgres_jupyterhub_dbname}}
+POSTGRES_USER={{postgres_jupyterhub_user}}
+POSTGRES_PASSWORD={{postgres_jupyterhub_password}}
+POSTGRES_HOST={{postgres_jupyterhub_host}}
 PGDATA=/var/lib/postgresql/data
 
+# Postgres settings for NBgrader
+POSTGRES_NBGRADER_HOST={{postgres_nbgrader_host}}
+POSTGRES_NBGRADER_USER={{postgres_nbgrader_user}}
+POSTGRES_NBGRADER_PASSWORD={{postgres_nbgrader_password}}
+POSTGRES_NBGRADER_PORT={{postgres_nbgrader_port}}
+POSTGRES_NBGRADER_DB={{postgres_nbgrader_dbname}}
+
 MNT_HOME_DIR_UID=1000
 MNT_HOME_DIR_GID=100
 MNT_ROOT={{mnt_root}}

ansible/roles/setup_course/files/requirements.txt

@@ -1,3 +1,4 @@
 docker-compose==1.26.2
 Hypercorn==0.10.1
 quart==0.12.0
+psycopg2-binary==2.8.6

ansible/roles/setup_course/templates/env.setup_course.j2

@@ -29,3 +29,10 @@ NB_UID=10001
 NB_GID=100
 
 SHARED_FOLDER_ENABLED={{shared_folder_enabled}}
+
+# Postgres settings for NBgrader
+POSTGRES_NBGRADER_HOST={{postgres_nbgrader_host}}
+POSTGRES_NBGRADER_USER={{postgres_nbgrader_user}}
+POSTGRES_NBGRADER_PASSWORD={{postgres_nbgrader_password}}
+POSTGRES_NBGRADER_PORT={{postgres_nbgrader_port}}
+POSTGRES_NBGRADER_DB={{postgres_nbgrader_dbname}}

src/illumidesk/apis/jupyterhub_api.py

@@ -1,11 +1,6 @@
 import json
 import os
 
-from nbgrader.api import Gradebook
-from nbgrader.api import InvalidEntry
-
-from pathlib import Path
-
 from tornado.httpclient import AsyncHTTPClient
 from tornado.httpclient import HTTPClientError
 from tornado.httpclient import HTTPResponse  # noqa: F401
@@ -146,39 +141,6 @@ async def add_group_member(self, group_name: str, username: str) -> Awaitable['H
             f'groups/{group_name}/users', body=json.dumps({'users': [f'{username}']}), method='POST',
         )
 
-    async def add_user_to_nbgrader_gradebook(
-        self, course_id: str, username: str, lms_user_id: str
-    ) -> Awaitable['HTTPResponse']:
-        """
-        Adds a user to the nbgrader gradebook database for the course.
-
-        Args:
-            course_id: The normalized string which represents the course label.
-            username: The user's username
-        Raises:
-            InvalidEntry: when there was an error adding the user to the database
-        """
-        if not course_id:
-            raise ValueError('course_id missing')
-        if not username:
-            raise ValueError('username missing')
-        if not lms_user_id:
-            raise ValueError('lms_user_id missing')
-        grader_name = f'grader-{course_id}'
-        db_url = Path('/home', grader_name, course_id, 'gradebook.db')
-        db_url.parent.mkdir(exist_ok=True, parents=True)
-        self.log.debug('Database url path is %s' % db_url)
-        if not db_url.exists():
-            self.log.debug('Gradebook database file does not exist')
-            return
-        gradebook = Gradebook(f'sqlite:///{db_url}', course_id=course_id)
-        try:
-            gradebook.update_or_create_student(username, lms_user_id=lms_user_id)
-            self.log.debug('Added user %s with lms_user_id %s to gradebook' % (username, lms_user_id))
-        except InvalidEntry as e:
-            self.log.debug('Error during adding student to gradebook: %s' % e)
-        gradebook.close()
-
     async def add_student_to_jupyterhub_group(self, course_id: str, student: str) -> Awaitable['HTTPResponse']:
         """
         Adds a student to the student course group.

src/illumidesk/apis/nbgrader_service.py

@@ -3,48 +3,73 @@
 import os
 import shutil
 
-from nbgrader.api import Assignment, Course, Gradebook
+from nbgrader.api import Assignment
+from nbgrader.api import Course
+from nbgrader.api import Gradebook
 from nbgrader.api import InvalidEntry
 
 from illumidesk.authenticators.utils import LTIUtils
 
+
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
 
 class NbGraderServiceHelper:
     """
-    Helper class to facilitate the use of nbgrader database and its methods
     """
-
     def __init__(self, course_id: str):
         if not course_id:
             raise ValueError('course_id missing')
 
         self.course_id = LTIUtils().normalize_string(course_id)
+        self.course_dir = f'/home/grader-{self.course_id}/{self.course_id}'
         self.uid = int(os.environ.get('NB_UID') or '10001')
         self.gid = int(os.environ.get('NB_GID') or '100')
-        grader_name = f'grader-{self.course_id}'
-        self.course_dir = f'/home/{grader_name}/{self.course_id}'
-        self.gradebook_path = Path(self.course_dir, 'gradebook.db')
-        # make sure the gradebook path exists
-        self.gradebook_path.parent.mkdir(exist_ok=True, parents=True)
-        logger.debug('Gradebook path is %s' % self.gradebook_path)
-        logger.debug("Creating gradebook instance")
-        # With new Gradebook instance the database is initiated/created
-        with Gradebook(f'sqlite:///{self.gradebook_path}', course_id=self.course_id):
-            logger.debug(
-                'Changing or making sure the gradebook directory permissions (with path %s) to %s:%s '
-                % (self.gradebook_path, self.uid, self.gid)
-            )
-            shutil.chown(str(self.gradebook_path), user=self.uid, group=self.gid)
+        self.org_name = os.environ.get('ORGANIZATION_NAME') or 'my-org'
+
+        # get nbgrader connection string from env vars
+        self.db_host = os.environ.get('POSTGRES_NBGRADER_HOST')
+        self.db_password = os.environ.get('POSTGRES_NBGRADER_PASSWORD')
+        self.db_port = os.environ.get('POSTGRES_NBGRADER_PORT')
+        self.db_name = os.environ.get('POSTGRES_NBGRADER_DB')
+        self.db_user = os.environ.get('POSTGRES_NBGRADER_USER')
+        self.db_url = f'postgresql://{self.db_user}:{self.db_password}@{self.db_host}:{self.db_port}/{self.db_name}'
+
+    def add_user_to_nbgrader_gradebook(self, username: str, lms_user_id: str) -> None:
+        """
+        Adds a user to the nbgrader gradebook database for the course.
+
+        Args:
+            username: The user's username
+            lms_user_id: The user's id on the LMS
+        Raises:
+            InvalidEntry: when there was an error adding the user to the database
+        """
+        if not username:
+            raise ValueError('username missing')
+        if not lms_user_id:
+            raise ValueError('lms_user_id missing')
+
+        with Gradebook(self.db_url, course_id=self.course_id) as gb:
+            try:
+                gb.update_or_create_student(username, lms_user_id=lms_user_id)
+                logger.debug('Added user %s with lms_user_id %s to gradebook' % (username, lms_user_id))
+            except InvalidEntry as e:
+                logger.debug('Error during adding student to gradebook: %s' % e)
 
     def update_course(self, **kwargs) -> None:
-        with Gradebook(f'sqlite:///{self.gradebook_path}', course_id=self.course_id) as gb:
+        """
+        Updates the course in nbgrader database
+        """
+        with Gradebook(self.db_url, course_id=self.course_id) as gb:
             gb.update_course(self.course_id, **kwargs)
 
     def get_course(self) -> Course:
-        with Gradebook(f'sqlite:///{self.gradebook_path}', course_id=self.course_id) as gb:
+        """
+        Gets the course model instance
+        """
+        with Gradebook(self.db_url, course_id=self.course_id) as gb:
             course = gb.check_course(self.course_id)
             logger.debug(f'course got from db:{course}')
             return course
@@ -63,7 +88,7 @@ def create_assignment_in_nbgrader(self, assignment_name: str, **kwargs: dict) ->
         assignment_name = LTIUtils().normalize_string(assignment_name)
         logger.debug('Assignment name normalized %s to save in gradebook' % assignment_name)
         assignment = None
-        with Gradebook(f'sqlite:///{self.gradebook_path}', course_id=self.course_id) as gb:
+        with Gradebook(self.db_url, course_id=self.course_id) as gb:
             try:
                 assignment = gb.update_or_create_assignment(assignment_name, **kwargs)
                 logger.debug('Added assignment %s to gradebook' % assignment_name)

src/illumidesk/authenticators/authenticator.py

@@ -67,11 +67,12 @@ async def setup_course_hook(
         raise EnvironmentError('ORGANIZATION_NAME env-var is not set')
     # normalize the name and course_id strings in authentication dictionary
     course_id = lti_utils.normalize_string(authentication['auth_state']['course_id'])
+    nb_service = NbGraderServiceHelper(course_id)
     username = lti_utils.normalize_string(authentication['name'])
     lms_user_id = authentication['auth_state']['lms_user_id']
     user_role = authentication['auth_state']['user_role']
     # register the user (it doesn't matter if it is a student or instructor) with her/his lms_user_id in nbgrader
-    await jupyterhub_api.add_user_to_nbgrader_gradebook(course_id, username, lms_user_id)
+    nb_service.add_user_to_nbgrader_gradebook(username, lms_user_id)
     # TODO: verify the logic to simplify groups creation and membership
     if user_is_a_student(user_role):
         # assign the user to 'nbgrader-<course_id>' group in jupyterhub and gradebook