Merge people service from integration into main

.github/workflows/python-source.yml

@@ -30,7 +30,7 @@ jobs:
         run: scripts/testall.docker python
 
       - name: Upload Artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: test-artifacts

docker/dockbuild.sh

@@ -23,7 +23,7 @@ PACKAGE_NAME=oar-pdr-py
 ## containers to be built.  List them in dependency order (where a latter one
 ## depends the former ones).  
 #
-DEP_DOCKER_IMAGE_DIRS="pymongo jqfromsrc ejsonschema pyenv"
+DEP_DOCKER_IMAGE_DIRS="pymongo jq ejsonschema pyenv"
 EXEC_DOCKER_IMAGE_DIRS="pdrpytest pdpserver midasserver peopleserver"
 
 [ -d "$codedir/metadata/docker" ] || {

docker/midasserver/README.md

@@ -1,33 +1,32 @@
 # MIDAS Web Services (for development mode)
 
 The intention of this Docker container is to run a fully functional MIDAS web service 
-suite<sup>*</sup> primarily for development and purposes.  It can be launched via the
-[`midasserver` script](https://github.com/usnistgov/oar-pdr-py/tree/feature/draft-service/scripts/midasserver)
-in the [`scripts` directory](https://github.com/usnistgov/oar-pdr-py/tree/feature/draft-service/scripts).
+suite primarily for development purposes.  It can be launched via the
+[`midasserver` script](../scripts/midasserver) in the [`scripts` directory](../scripts).
 
 By default, the server operates with a storage backend in which records are stored in JSON
 files beneath a specified data directory.  This makes it easy to inspect the current contents
 of the stored records during development.  However, the server can optionally be run using a
 MongoDB backend.
 
-<sup>*</sup>_Note: While the server is designed to provided the full suite of MIDAS APIs,
-as of this writing, only the DMP project service is available._
-
 ## Prerequisites for running the server
 
 To run this server "out of the box" requires:
 
-  * Python 3 (>= 3.8.X)
+  * Python 3 (>= 3.10.X)
   * Docker Engine with command-line tools
     * if you want to use the option MongoDB backend, you will also need the [docker compose
       plugin](https://docs.docker.com/get-started/08_using_compose/).  This is included with
       Docker Desktop (typical for Mac installs); to check to see if you already have it, type,
       `docker compose version`.
   * The [oar-pdr-py repository](https://github.com/usnistgov/oar-pdr-py) cloned onto your machine.
 
+It is not necessary to build the python software separately; this can be done on-the-fly the first
+time your script run the server script.
+
 ## Starting and stopping the server
 
-To run the server, you should open a terminal and change into you local clone of the `oar-pdr-py`
+To run the server, you should open a terminal and change into your local clone of the `oar-pdr-py`
 repository.  The server is launched using the `midasserver` script located in the `scripts`
 directory.  Note when you run the script for the first time, it will automatically build all of
 the python code and docker images (producing a lot of output to the screen); these are not rebuilt
@@ -37,7 +36,7 @@ To start the server, you provide the name of the directory where you want the ba
 For example, you can type:
 
 ```bash
-scripts/midasserver midasdata
+scripts/midasserver midasdata --bg
 ```
 
 This will create a `midasdata` subdirectory in the current directory.  The actual record data will
@@ -78,6 +77,19 @@ shutdown as well:
 scripts/midasserver -M stop
 ```
 
+### Launching with staff directory APIs included
+
+If the server is started with the `-P` (or `--add-people-service`) option, the staff directory
+service APIs will be included among the MIDAS APIs.
+
+```bash
+scripts/midasserver -M -P midasdata
+```
+
+These will be available under the `http://localhost:9091/nsd` endpoint.  See the
+[peopleservice README documentation](../peopleserver/README.md) for details about using these
+APIs. 
+
 ## Using the service
 
 The base URLs for the MIDAS services are as follows:

docker/midasserver/midas-dmpdap_conf.yml

@@ -83,3 +83,20 @@ services:
                 describedBy: "http://localhost:9091/docs/dmpsvc-elements.html"
                 href:        "http://localhost:9091/midas/dmp/mdm1"
                 version:     mdm1
+
+   nsdi:
+      about:
+         message:     "NSD Indexing Service"
+         title:       "NIST Staff Directory Indexing API"
+         describedBy: "http://localhost:9091/docs/nsdisvc-elements.html"
+         href:        "http://localhost:9091/midas/nsdi"
+      default_convention: v1
+      conventions: 
+         v1:
+            nsd:
+                service_endpoint: http://peopleserver:9092/
+            about: 
+                title:       "NIST Staff Directory Indexing API (version 1)"
+                describedBy: "http://localhost:9091/docs/nsdisvc-elements.html"
+                href:        "http://localhost:9091/midas/nsdi/v1"
+                version:     v1

docker/midasserver/midas-dmpdapnsd_conf.yml

@@ -0,0 +1,109 @@
+logfile: midas.log
+loglevel: DEBUG
+dbio:
+   factory:  fsbased
+about: 
+   title: "MIDAS Authoring Services"
+   describedBy: "http://localhost:9091/midas/docs"
+   href: "http://localhost:9091/midas/"
+services: 
+   dap:
+      about: 
+         message:     "DMP Service is available"
+         title:       "Digital Asset Publication (DAP) Authoring API"
+         describedBy: "http://localhost:9091/docs/dapsvc-elements.html"
+         href:        "http://localhost:9091/midas/dap"
+
+      clients: 
+         midas: 
+            default_shoulder: mds3
+         default: 
+            default_shoulder: mds3
+
+      dbio: 
+         superusers:                [ "rlp3" ]
+         allowed_project_shoulders: ["mdsx", "mds3", "mds0", "pdr0"]
+         default_shoulder:          mdsx
+
+      include_headers:
+         "Access-Control-Allow-Origin": "*"
+
+      default_convention: mds3
+      conventions: 
+         mdsx: 
+            about: 
+                title:       "Digital Asset Publication (DAP) Authoring API (experimental)"
+                describedBy: "http://localhost:9091/docs/dapsvc-elements.html"
+                href:        "http://localhost:9091/midas/dap/mdsx"
+                version:     mdsx
+            assign_doi: always
+            doi_naan:   "10.18434"
+         mds3: 
+            about: 
+                title:       "Digital Asset Publication (DAP) Authoring API (mds3 convention)"
+                describedBy: "http://localhost:9091/docs/dapsvc-elements.html"
+                href:        "http://localhost:9091/midas/dap/mds3"
+                version:     mds3
+            assign_doi: always
+            doi_naan:   "10.18434"
+            nerdstorage:
+              type: fsbased
+              store_dir: /data/midas/nerdm
+
+   dmp: 
+      about: 
+         message:     "DMP Service is available"
+         title:       "Data Management Plan (DMP) Authoring API"
+         describedBy: "http://localhost:9091/docs/dmpsvc-elements.html"
+         href:        "http://localhost:9091/midas/dmp"
+
+      clients: 
+          midas: 
+             default_shoulder: mdm1
+          default: 
+             default_shoulder: mdm1
+
+      dbio: 
+         superusers:                [ "rlp3" ]
+         allowed_project_shoulders: ["mdm0", "mdm1"]
+         default_shoulder:          mdm1
+
+      include_headers:
+         "Access-Control-Allow-Origin": "*"
+
+      default_convention: mdm1
+      conventions: 
+         mdm1: 
+            about: 
+                title:       "Data Management Plan (DMP) Authoring API (mdm1 convention)"
+                describedBy: "http://localhost:9091/docs/dmpsvc-elements.html"
+                href:        "http://localhost:9091/midas/dmp/mdm1"
+                version:     mdm1
+
+   nsd:
+      about:
+         message:     "NSD Service"
+         title:       "NIST Staff Directory Service API"
+         describedBy: "http://localhost:9091/docs/nsdsvc-elements.html"
+         href:        "http://localhost:9091/midas/nsd"
+      include_headers:
+         "Access-Control-Allow-Origin": "*"
+      default_convention: nsd1
+      db_url: "mongodb://oarop:oarop@mongodb:27017/midas"
+      data:
+        dir: /data/nsd
+        person_file: person.json
+        org_file: orgs.json
+      conventions: 
+         nsd1:
+            about: 
+                title:       "NIST Staff Directory Service API (NSD version 1)"
+                describedBy: "http://localhost:9091/docs/nsdsvc-elements.html"
+                href:        "http://localhost:9091/midas/nsd/nsd1"
+                version:     nsd1
+         oar1:
+            about: 
+                title:       "NIST Staff Directory Service API (OAR version 1)"
+                describedBy: "http://localhost:9091/docs/nsdsvc-elements.html"
+                href:        "http://localhost:9091/midas/nsd/oar1"
+                version:     oar1

docker/midasserver/mongo/mongo-init.sh

@@ -8,4 +8,4 @@ echo '
             roles: [ "readWrite" ]
         }
     )
-    exit' | mongo -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD
+    exit' | mongosh -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD

docker/midasserver/run.sh

@@ -15,6 +15,7 @@ SED_RE_OPT=r
 
 PACKAGE_NAME=oar-pdr-py
 DEFAULT_CONFIGFILE=$dockerdir/midasserver/midas-dmpdap_conf.yml
+NSD_CONFIGFILE=$dockerdir/midasserver/midas-dmpdapnsd_conf.yml
 
 set -e
 
@@ -51,6 +52,9 @@ ARGUMENTS
   -M, --use-mongodb             Use a MongoDB backend; DIR must also be provided.
                                 If not set, a file-based database (using JSON 
                                 files) will be used, stored under DIR/dbfiles.
+  -P, --add-people-service      Include the staff directory service within the application.
+                                This will trigger use of a MongoDB database, but it 
+                                does not effect the DBIO backend (use -M for this).  
   -p, --port NUM                The port that the service should listen to 
                                 (default: 9091)
   -h, --help                    Print this text to the terminal and then exit
@@ -79,7 +83,10 @@ CONFIGFILE=
 USEMONGO=
 STOREDIR=
 DBTYPE=
+ADDNSD=
 DETACH=
+PPLDATADIR=
+VOLOPTS="-v $repodir/dist:/app/dist"
 while [ "$1" != "" ]; do
     case "$1" in
         -b|--build)
@@ -108,6 +115,25 @@ while [ "$1" != "" ]; do
         -M|--use-mongo)
             DBTYPE="mongo"
             ;;
+        -P|--add-people-service)
+            ADDNSD=1
+            ;;
+        -d|--people-data-dir)
+            shift
+            PPLDATADIR=$1
+            ;;
+        --people-data-dir=*)
+            PPLDATADIR=`echo $1 | sed -e 's/[^=]*=//'`
+            ;;
+        --mount-volume=*)
+            vol=`echo $1 | sed -e 's/[^=]*=//'`
+            VOLOPTS="$VOLOPTS -v $vol"
+            ;;
+        -V)
+            shift
+            vol=$1
+            VOLOPTS="$VOLOPTS -v $vol"
+            ;;
         -h|--help)
             usage
             exit
@@ -133,6 +159,12 @@ while [ "$1" != "" ]; do
     esac
     shift
 done
+if [ -z "$PPLDATADIR" ]; then
+    PPLDATADIR=$repodir/python/tests/nistoar/nsd/data
+else
+    PPLDATADIR=`(cd $PPLDATADIR > /dev/null 2>&1; pwd)`
+fi
+[ -n "$PPLDATADIR" ] || 
 [ -n "$ACTION" ] || ACTION=start
 
 ([ -z "$DOPYBUILD" ] && [ -e "$repodir/dist/pdr" ]) || {
@@ -143,12 +175,23 @@ done
     echo ${prog}: Python library not found in dist directory: $repodir/dist
     false
 }
-VOLOPTS="-v $repodir/dist:/app/dist"
+
+[ -n "$PPLDATADIR" ] || PPLDATADIR=$repodir/docker/peopleserver/data
+ls $PPLDATADIR/*.json > /dev/null 2>&1 || {
+    # no JSON data found in datadir; reset the datadir to test data
+    echo "${prog}: no people data found; will load db with test data"
+    PPLDATADIR=$repodir/python/tests/nistoar/nsd/data
+}
+[ "$ACTION" = "stop" ] || echo "${prog}: loading staff directory DB from $PPLDATADIR"
+VOLOPTS="$VOLOPTS -v ${PPLDATADIR}:/data/nsd"
 
 # build the docker images if necessary
 (docker_images_built midasserver && [ -z "$DODOCKBUILD" ]) || build_server_image
 
-[ -n "$CONFIGFILE" ] || CONFIGFILE=$DEFAULT_CONFIGFILE
+[ -n "$CONFIGFILE" ] || {
+    CONFIGFILE=$DEFAULT_CONFIGFILE
+    [ -z "$ADDNSD" ] || CONFIGFILE=$NSD_CONFIGFILE
+}
 [ -f "$CONFIGFILE" ] || {
     echo "${prog}: Config file ${CONFIGFILE}: does not exist as a file"
     false
@@ -184,7 +227,7 @@ fi
 
 NETOPTS=
 STOP_MONGO=true
-if [ "$DBTYPE" = "mongo" ]; then
+if [ "$DBTYPE" = "mongo" -o -n "$ADDNSD" ]; then
     DOCKER_COMPOSE="docker compose"
     (docker compose version > /dev/null 2>&1) || DOCKER_COMPOSE=docker-compose
     ($DOCKER_COMPOSE version  > /dev/null 2>&1) || {
@@ -196,7 +239,7 @@ if [ "$DBTYPE" = "mongo" ]; then
     source $dockerdir/midasserver/mongo/mongo.env
 
     [ -n "$STOREDIR" -o "$ACTION" = "stop" ] || {
-        echo ${prog}: DIR argument must be provided with -M/--use-mongo
+        echo ${prog}: DIR argument must be provided with -M/--use-mongo or -P/--add-people-service
         false
     }
     export OAR_MONGODB_DBDIR=`cd $STOREDIR; pwd`/mongo