Complete integration of people service into MIDAS by enabling loading

of people data into database:
  * provide load functions to underlying apps
  * leverage load function at start-up time
  * add LOAD method to MIDAS app for updating people data into
    running system.

docker/midasserver/midas-dmpdapnsd_conf.yml

@@ -0,0 +1,105 @@
+logfile: midas.log
+loglevel: DEBUG
+dbio:
+   factory:  fsbased
+about: 
+   title: "MIDAS Authoring Services"
+   describedBy: "http://localhost:9091/midas/docs"
+   href: "http://localhost:9091/midas/"
+services: 
+   dap:
+      about: 
+         message:     "DMP Service is available"
+         title:       "Digital Asset Publication (DAP) Authoring API"
+         describedBy: "http://localhost:9091/docs/dapsvc-elements.html"
+         href:        "http://localhost:9091/midas/dap"
+
+      clients: 
+         midas: 
+            default_shoulder: mds3
+         default: 
+            default_shoulder: mds3
+
+      dbio: 
+         superusers:                [ "rlp3" ]
+         allowed_project_shoulders: ["mdsx", "mds3", "mds0", "pdr0"]
+         default_shoulder:          mdsx
+
+      include_headers:
+         "Access-Control-Allow-Origin": "*"
+
+      default_convention: mds3
+      conventions: 
+         mdsx: 
+            about: 
+                title:       "Digital Asset Publication (DAP) Authoring API (experimental)"
+                describedBy: "http://localhost:9091/docs/dapsvc-elements.html"
+                href:        "http://localhost:9091/midas/dap/mdsx"
+                version:     mdsx
+            assign_doi: always
+            doi_naan:   "10.18434"
+         mds3: 
+            about: 
+                title:       "Digital Asset Publication (DAP) Authoring API (mds3 convention)"
+                describedBy: "http://localhost:9091/docs/dapsvc-elements.html"
+                href:        "http://localhost:9091/midas/dap/mds3"
+                version:     mds3
+            assign_doi: always
+            doi_naan:   "10.18434"
+            nerdstorage:
+              type: fsbased
+              store_dir: /data/midas/nerdm
+
+   dmp: 
+      about: 
+         message:     "DMP Service is available"
+         title:       "Data Management Plan (DMP) Authoring API"
+         describedBy: "http://localhost:9091/docs/dmpsvc-elements.html"
+         href:        "http://localhost:9091/midas/dmp"
+
+      clients: 
+          midas: 
+             default_shoulder: mdm1
+          default: 
+             default_shoulder: mdm1
+
+      dbio: 
+         superusers:                [ "rlp3" ]
+         allowed_project_shoulders: ["mdm0", "mdm1"]
+         default_shoulder:          mdm1
+
+      include_headers:
+         "Access-Control-Allow-Origin": "*"
+
+      default_convention: mdm1
+      conventions: 
+         mdm1: 
+            about: 
+                title:       "Data Management Plan (DMP) Authoring API (mdm1 convention)"
+                describedBy: "http://localhost:9091/docs/dmpsvc-elements.html"
+                href:        "http://localhost:9091/midas/dmp/mdm1"
+                version:     mdm1
+
+   nsd:
+      about:
+         message:     "NSD Service"
+         title:       "NIST Staff Directory Service API"
+         describedBy: "http://localhost:9091/docs/nsdsvc-elements.html"
+         href:        "http://localhost:9091/midas/nsd"
+      default_convention: nsd1
+      db_url: "mongodb://oarop:oarop@mongodb:27017/midas"
+      data:
+        dir: /data/nsd
+      conventions: 
+         nsd1:
+            about: 
+                title:       "NIST Staff Directory Service API (NSD version 1)"
+                describedBy: "http://localhost:9091/docs/nsdsvc-elements.html"
+                href:        "http://localhost:9091/midas/nsd/nsd1"
+                version:     nsd1
+         oar1:
+            about: 
+                title:       "NIST Staff Directory Service API (OAR version 1)"
+                describedBy: "http://localhost:9091/docs/nsdsvc-elements.html"
+                href:        "http://localhost:9091/midas/nsd/oar1"
+                version:     oar1

docker/midasserver/run.sh

@@ -15,6 +15,7 @@ SED_RE_OPT=r
 
 PACKAGE_NAME=oar-pdr-py
 DEFAULT_CONFIGFILE=$dockerdir/midasserver/midas-dmpdap_conf.yml
+NSD_CONFIGFILE=$dockerdir/midasserver/midas-dmpdapnsd_conf.yml
 
 set -e
 
@@ -51,6 +52,9 @@ ARGUMENTS
   -M, --use-mongodb             Use a MongoDB backend; DIR must also be provided.
                                 If not set, a file-based database (using JSON 
                                 files) will be used, stored under DIR/dbfiles.
+  -P, --add-people-service      Include the staff directory service within the application.
+                                This will trigger use of a MongoDB database, but it 
+                                does not effect the DBIO backend (use -M for this).  
   -p, --port NUM                The port that the service should listen to 
                                 (default: 9091)
   -h, --help                    Print this text to the terminal and then exit
@@ -79,7 +83,9 @@ CONFIGFILE=
 USEMONGO=
 STOREDIR=
 DBTYPE=
+ADDNSD=
 DETACH=
+VOLOPTS="-v $repodir/dist:/app/dist"
 while [ "$1" != "" ]; do
     case "$1" in
         -b|--build)
@@ -108,6 +114,18 @@ while [ "$1" != "" ]; do
         -M|--use-mongo)
             DBTYPE="mongo"
             ;;
+        -P|--add-people-service)
+            ADDNSD=1
+            ;;
+        --mount-volume=*)
+            vol=`echo $1 | sed -e 's/[^=]*=//'`
+            VOLOPTS="$VOLOPTS -v $vol"
+            ;;
+        -V)
+            vol=$1
+            shift
+            VOLOPTS="$VOLOPTS -v $vol"
+            ;;
         -h|--help)
             usage
             exit
@@ -143,12 +161,14 @@ done
     echo ${prog}: Python library not found in dist directory: $repodir/dist
     false
 }
-VOLOPTS="-v $repodir/dist:/app/dist"
 
 # build the docker images if necessary
 (docker_images_built midasserver && [ -z "$DODOCKBUILD" ]) || build_server_image
 
-[ -n "$CONFIGFILE" ] || CONFIGFILE=$DEFAULT_CONFIGFILE
+[ -n "$CONFIGFILE" ] || {
+    CONFIGFILE=$DEFAULT_CONFIGFILE
+    [ -z "$ADDNSD" ] || CONFIGFILE=$NSD_CONFIGFILE
+}
 [ -f "$CONFIGFILE" ] || {
     echo "${prog}: Config file ${CONFIGFILE}: does not exist as a file"
     false
@@ -184,7 +204,7 @@ fi
 
 NETOPTS=
 STOP_MONGO=true
-if [ "$DBTYPE" = "mongo" ]; then
+if [ "$DBTYPE" = "mongo" -o -n "$ADDNSD" ]; then
     DOCKER_COMPOSE="docker compose"
     (docker compose version > /dev/null 2>&1) || DOCKER_COMPOSE=docker-compose
     ($DOCKER_COMPOSE version  > /dev/null 2>&1) || {
@@ -196,7 +216,7 @@ if [ "$DBTYPE" = "mongo" ]; then
     source $dockerdir/midasserver/mongo/mongo.env
 
     [ -n "$STOREDIR" -o "$ACTION" = "stop" ] || {
-        echo ${prog}: DIR argument must be provided with -M/--use-mongo
+        echo ${prog}: DIR argument must be provided with -M/--use-mongo or -P/--add-people-service
         false
     }
     export OAR_MONGODB_DBDIR=`cd $STOREDIR; pwd`/mongo

python/nistoar/midas/wsgi.py

@@ -577,8 +577,12 @@ def handle_path_request(self, path: str, env: Mapping, start_resp: Callable, who
 
         return subapp.handle_path_request(env, start_resp, "/".join(path), who)
 
-    def load_people_from(self, datadir):
+    def load_people_from(self, datadir=None):
         if any(k.startswith("nsd/") for k in self.subapps.keys()):
+            if not datadir:
+                nsdcfg = self.cfg.get('services',{}).get("nsd")
+                datadir = nsdcfg.get("data", {}).get("dir", "/data/nsd")
+
             nsdapp = [v for k,v in self.subapps.items() if k.startswith("nsd/")][0]
             nsdapp.load_from(datadir)
 

python/nistoar/nsd/service.py

@@ -253,10 +253,28 @@ def load_orgs(self, data: List[Mapping]):
         self._db['Orgs'].insert_many(data)
 
 
-    def load(self, config, log=None, clear=True):
+    def load(self, config, log=None, clear=True, withtrans=False):
         """
         load all data described in the given configuration object.  This is done safely within a
         database transaction so that it rolls back if there is a failure.  
+
+        The configuration provided to this function is used to control the loading.  In particular,
+        the following parameters will be looked for:
+
+        ``dir``
+            the directory where data files to be loaded should be found (default: "/data/nsd").
+        ``person_file``
+            the name of the file containing records describing people
+        ``org_file``
+            the name of the file containing records describing that organizations the people are 
+            assigned to.
+
+        :param dict config:  the configuration to use during loading (see above)
+        :param Logger  log:  the Logger to send messages to
+        :param bool  clear:  if True, all previously loaded records in the database will be deleted 
+                             before loading (default: True)
+        :param bool withtrans:  if True, use a database transaction to do the loading.  (Note this 
+                             requires that the MongoDB be started with replicaSets; default: False.)
         """
         datadir = config.get('dir', '.')
         if not os.path.isdir(datadir):
@@ -268,32 +286,44 @@ def load(self, config, log=None, clear=True):
         if not os.path.isfile(os.path.join(datadir, orgfile)):
             raise ConfigurationException(f"{orgfile}: NSD data directory does not exist as a file")
 
+        if not withtrans:
+            self._load_notrans(datadir, personfile, orgfile, log, clear)
+        else:
+            self._load_notrans(datadir, personfile, orgfile, log, clear)
+
+    def _load_notrans(self, datadir, personfile, orgfile, log, clear):
+        people = self._db.People
+        orgs = self._db.Orgs
+
+        if clear:
+            people.delete_many({})
+            orgs.delete_many({})
+
+        self._load_file(people, personfile, datadir, log)
+        self._load_file(orgs, orgfile, datadir, log)
+
+    def _load_withtrans(self, datadir, personfile, orgfile, log, clear):
+
         def _loadit(session):
             people = session.client[self._db.name].People
             orgs   = session.client[self._db.name].Orgs
 
             if clear:
-#                self._db['OUs'].delete_many({})
-#                self._db['Divisions'].delete_many({})
-#                self._db['Groups'].delete_many({})
-                self._db['People'].delete_many({})
-                self._db['Orgs'].delete_many({})
-
-#            self._load_file(self.load_OUs, config.get('ou_file', "ou.json"), datadir, log)
-#            self._load_file(self.load_divs, config.get('div_file', "div.json"), datadir, log)
-#            self._load_file(self.load_groups, config.get('group_file', "group.json"), datadir, log)
-            self._load_file(self.load_people, personfile, datadir, log)
-            self._load_file(self.load_orgs, orgfile, datadir, log)
+                people.delete_many({}, session=session)
+                orgs.delete_many({}, session=session)
+
+            self._load_file(people, personfile, datadir, log, session)
+            self._load_file(orgs, orgfile, datadir, log, session)
 
         with self._cli.start_session() as session:
             session.with_transaction(_loadit)
 
-    def _load_file(self, loader, file, dir='.', log=None):
+    def _load_file(self, mongocoll, file, dir='.', log=None, session=None):
         try:
             datafile = os.path.join(dir, file)
             with open(datafile) as fd:
                 data = json.load(fd)
-            loader(data)
+            mongocoll.insert_many(data, session=session)
         except FileNotFoundError as ex:
             if log:
                 log.warning("Source data file not found: %s", datafile)

python/nistoar/nsd/wsgi/nsd1.py

@@ -27,7 +27,7 @@ def __init__(self, service: PeopleService, path: str, wsgienv: Mapping, start_re
                  who=None, config: Mapping={}, log: logging.Logger=None, app=None):
         if not log:
             log = deflog
-        super(NSDHandler, self).__init__(path, wsgienv, start_resp, who, config, log)
+        super(NSDHandler, self).__init__(path, wsgienv, start_resp, who, config, log, app)
 
         self.svc = service
         self._format_qp = "format"

python/nistoar/nsd/wsgi/oar1.py

@@ -352,7 +352,23 @@ def do_GET(self, path, ashead=False, format=None):
         except UnsupportedFormat as ex:
             return self.send_error(400, "Unsupported Format", str(ex))
 
-        return self.send_json(self.svc.status(), ashead=ashead)
+        out = self.svc.status()
+        if out.get("status") == "ready":
+            return self.send_json(out, ashead=ashead)
+        return self.send_json(out, "Not Ready", 503, ashead=ashead)
+
+    def do_LOAD(self, path):
+        if self.who.actor not in self.cfg.get("loaderusers", []):
+            self.log.warning("Unauthorized user attempted reload NSD data: %s", self.who.actor)
+            return self.send_error_obj(401, "Not Authorized",
+                                       f"{self.who.actor} is not authorized to reload")
+
+        try:
+            self.app.load_from()
+            return self.send_error_obj(200, "Data Reloaded", "Successfully reloaded NSD data")
+        except Exception as ex:
+            self.log.error("Failed to reload NSD data: %s", str(ex))
+            return self.send_error_obj(500, "Internal Server Error")
 
     def do_OPTIONS(self, path):
         return self.send_options(["GET"])
@@ -374,29 +390,39 @@ def __init__(self, config, log, appname=None):
 
         self.svc = MongoPeopleService(dburl)
 
-    def load_from(self, datadir):
+    def load_from(self, datadir=None):
         """
-        initialize the people database from files in the given data directory
+        initialize the people database.  This will use the configuration under the ``data`` 
+        parameter to control the loading (see :py:class:`~nistoar.nsd.service.PeopleService` 
+        supported parameters).
+
+        :param str datadir:   the directory to look for data files in, overriding the value
+                              provided in the configuration.  
         """
-        self.svc.load({"dir": datadir}, self.log, True)
+        datacfg = self.cfg.get("data", {})
+        datacfg.setdefault("dir", "/data/nsd")
+        if datadir:
+            datacfg = dict(self.cfg.items())
+            datacfg['datadir'] = datadir
+        self.svc.load(datacfg, self.log, True)
 
     def create_handler(self, env, start_resp, path, who) -> Handler:
         parts = path.split('/', 1)
         what = parts.pop(0).lower()
         path = parts[0] if parts else ""
 
         if not what:
-            return ReadyHandler(self.svc, path, env, start_resp, who, log=self.log, app=self)
+            return ReadyHandler(self.svc, path, env, start_resp, who, self.cfg, log=self.log, app=self)
 
         if what == "people":
-            return PeopleHandler(self.svc, path, env, start_resp, who, log=self.log.getChild("people"),
-                                 app=self)
+            return PeopleHandler(self.svc, path, env, start_resp, who, self.cfg, 
+                                 log=self.log.getChild("people"), app=self)
 
         if what == "orgs":
-            return OrgHandler(self.svc, path, env, start_resp, who, log=self.log.getChild("people"),
-                              app=self)
+            return OrgHandler(self.svc, path, env, start_resp, who, self.cfg, 
+                              log=self.log.getChild("people"), app=self)
 
-        return NotFoundHandler(path, env, start_resp, log=self.log, app=self)
+        return NotFoundHandler(path, env, start_resp, self.cfg, log=self.log, app=self)
 
 
 class PeopleApp(WSGIServiceApp):
@@ -434,5 +460,11 @@ def authenticate_user(self, env: Mapping, agents: List[str]=None, client_id: str
         authcfg = self.cfg.get('authentication', {})
         return authenticate_via_jwt("nsd", env, authcfg, self.log, agents, client_id)
 
+    def load(self):
+        """
+        (re-)initialize the underlying database with data from the configured data directory
+        """
+        self.svcapps[''].load_from()
+
 
 app = PeopleApp