Merge integration into main for 2.1.0 release

docker/cacerts/README.md

@@ -0,0 +1,13 @@
+This directory contains non-standard CA certificates needed to build the docker
+images. 
+
+Failures building the Docker containers defined in ../ due to SSL certificate
+verification errors may be a consequence of your local network's firewall.  In
+particular, the firewall may be substituting external site certificates with
+its own signed by a non-standard CA certficate (chain).  If so, you can place 
+the necessary certificates into this directory; they will be passed into the
+containers, allowing them to safely connect to those external sites.
+
+Be sure the certificates are in PEM format and include a .crt file extension.
+
+Do not remove this README file; doing so may cause a Docker build faiure.

docker/dockbuild.sh

@@ -23,14 +23,15 @@ PACKAGE_NAME=oar-pdr-py
 ## containers to be built.  List them in dependency order (where a latter one
 ## depends the former ones).  
 #
-DOCKER_IMAGE_DIRS="pymongo jqfromsrc ejsonschema pyenv pdrpytest pdpserver"
+DEP_DOCKER_IMAGE_DIRS="pymongo jqfromsrc ejsonschema pyenv"
+EXEC_DOCKER_IMAGE_DIRS="pdrpytest pdpserver midasserver"
 
-[ -d "$codedir/metadata/oar-build" ] || {
+[ -d "$codedir/metadata/docker" ] || {
     echo ${prog}: Missing metadata submodule
     echo Clone the oar-metadata repo in this directory\; name it "'metadata'"
     exit 3
 }
-. $codedir/metadata/oar-build/_dockbuild.sh
+. $codedir/oar-build/_dockbuild.sh
 
 # Override, if need be, the UID of the user to run as in the container; the 
 # default is the user running this script.
@@ -43,6 +44,9 @@ setup_build
 
 log_intro   # record start of build into log
 
+if { echo $BUILD_IMAGES | grep -qs pymongo; }; then
+    cp_ca_certs_to ../metadata/docker
+fi
 $codedir/metadata/docker/dockbuild.sh $BUILD_IMAGES
 
 if { echo " $BUILD_IMAGES " | grep -qs " pyenv "; }; then
@@ -57,3 +61,7 @@ if { echo " $BUILD_IMAGES " | grep -qs " pdpserver "; }; then
     echo '+' docker build $BUILD_OPTS -t $PACKAGE_NAME/pdpserver pdpserver | logit
     docker build $BUILD_OPTS -t $PACKAGE_NAME/pdpserver pdpserver 2>&1 | logit
 fi
+if { echo " $BUILD_IMAGES " | grep -qs " midasserver "; }; then
+    echo '+' docker build $BUILD_OPTS -t $PACKAGE_NAME/midasserver midasserver | logit
+    docker build $BUILD_OPTS -t $PACKAGE_NAME/midasserver midasserver 2>&1 | logit
+fi

docker/midasserver/Dockerfile

@@ -0,0 +1,24 @@
+#########################################################################
+#
+# MIDAS Authoring web service suite
+#
+# This container launches the MIDAS web services via scripts/midas-uwsgi.py
+#
+#########################################################################
+FROM oar-pdr-py/pyenv
+
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod a+rx /usr/local/bin/entrypoint.sh
+
+VOLUME /dev/oar-pdr-py
+VOLUME /app/dist
+
+RUN mkdir -p /dev/oar-pdr-py /app && chmod a+rwx /app
+WORKDIR /dev/oar-pdr-py
+
+ENV PYTHONPATH /dev/oar-pdr-py/python/dist/pdr/lib/python:/app/dist/pdr/lib/python
+
+ARG devuser=developer
+USER $devuser
+EXPOSE 9091
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker/midasserver/README.md

@@ -0,0 +1,218 @@
+# MIDAS Web Services (for development mode)
+
+The intention of this Docker container is to run a fully functional MIDAS web service 
+suite<sup>*</sup> primarily for development and purposes.  It can be launched via the
+[`midasserver` script](https://github.com/usnistgov/oar-pdr-py/tree/feature/draft-service/scripts/midasserver)
+in the [`scripts` directory](https://github.com/usnistgov/oar-pdr-py/tree/feature/draft-service/scripts).
+
+By default, the server operates with a storage backend in which records are stored in JSON
+files beneath a specified data directory.  This makes it easy to inspect the current contents
+of the stored records during development.  However, the server can optionally be run using a
+MongoDB backend.
+
+<sup>*</sup>_Note: While the server is designed to provided the full suite of MIDAS APIs,
+as of this writing, only the DMP project service is available._
+
+## Prerequisites for running the server
+
+To run this server "out of the box" requires:
+
+  * Python 3 (>= 3.8.X)
+  * Docker Engine with command-line tools
+    * if you want to use the option MongoDB backend, you will also need the [docker compose
+      plugin](https://docs.docker.com/get-started/08_using_compose/).  This is included with
+      Docker Desktop (typical for Mac installs); to check to see if you already have it, type,
+      `docker compose version`.
+  * The [oar-pdr-py repository](https://github.com/usnistgov/oar-pdr-py) cloned onto your machine.
+
+## Starting and stopping the server
+
+To run the server, you should open a terminal and change into you local clone of the `oar-pdr-py`
+repository.  The server is launched using the `midasserver` script located in the `scripts`
+directory.  Note when you run the script for the first time, it will automatically build all of
+the python code and docker images (producing a lot of output to the screen); these are not rebuilt
+by default on subsequent executions.
+
+To start the server, you provide the name of the directory where you want the backend data written.
+For example, you can type:
+
+```bash
+scripts/midasserver midasdata
+```
+
+This will create a `midasdata` subdirectory in the current directory.  The actual record data will
+be stored under `./midasdata/dbfiles`.  
+
+The server runs in a docker container in the shell's forground; that is, you will not get your shell
+prompt back.  This allows you to see messages logging each call made to the service.  To stop the
+server, you need to open another terminal, change into the `oar-pdr-py` repository directory, and
+type:
+
+```bash
+scripts/midasserver stop
+```
+
+More information about `midasserver` command-line options can be viewed via the `-h` option:
+
+```bash
+scripts/midasserver -h
+```
+
+### Launching with a MongoDB backend
+
+The server can be optionally switched to store its records in a MongoDB database backend with the
+`-M` command-line option:
+
+```bash
+scripts/midasserver -M midasdata
+```
+
+In addition to starting a MongoDB database server in a Docker container, it also launches a server
+frontend (called mongo-express) that allows you to explore the database contents via a web browser.  
+To view, open the URL, `http://localhost:8081/`.
+
+To stop the server, be sure to also provide the `-M` option to ensure that the Mongo database gets
+shutdown as well:
+
+```bash
+scripts/midasserver -M stop
+```
+
+## Using the service
+
+The base URLs for the MIDAS services are as follows:
+
+  - [`http://localhost:9091/midas/dmp/mdm1`](http://localhost:9091/midas/dmp/mdm1) -- the Data
+    Management Plan (DMP) project service
+  - `http://localhost:9091/midas/dap/mds3` -- the Digital Asset Publication (DAP) project service
+  - `http://localhost:9091/midas/groups`   -- the MIDAS user group service
+
+Note that when you start the service, you also have access to online API documentation.  To view,
+open the URL,
+[`http://localhost:9091/docs/dmpsvc-elements.html`](http://localhost:9091/docs/dmpsvc-elements.html).
+
+At this time, all request and response messages are JSON documents with a content type of
+"application/json".  This content type is assumed by default, so "Accept" and "Content-Header"
+HTTP headers are not required. 
+
+### Creating a new DMP project
+
+Creating a new DMP project record is done by POSTing to the service's base URL.  The request
+body must be a JSON object which can contain the following properties:
+
+  - `name` -- (Required)  The user-supplied mnemonic name to assign to the new record.  This name
+    is intended only for display purposes; it is not part of the DMP data content.
+  - `data` -- (Optional)  A JSON object containing the data content.  Each project type will
+    enforce its own schema for this object.  The data included here is not expected to be complete.
+  - `meta` -- (Optional)  A JSON object containing metadata hints that help the server manage
+    the record.  This data will not be part of the public DMP data content.  Creation is the only
+    time the client can directly add information to this object; although the server may update
+    the information triggered by other user requests.  Unrecognized data in this object may be
+    ignored.  
+
+For example, an initial record might be created with:
+
+```bash
+curl -X POST --data '{"name": "CoTEM", "data": {"title": "Microscopy of Cobalt Samples"}}' \
+     http://localhost:9091/midas/dmp/mdm1
+```
+
+If the creation request is successful, the request will return a 201 status and a JSON document
+containing the full, newly created record:
+
+```json
+{
+  "id": "mdm1:0003",
+  "name": "CoTEM",
+  "acls": {
+    "read": [
+      "anonymous"
+    ],
+    "write": [
+      "anonymous"
+    ],
+    "admin": [
+      "anonymous"
+    ],
+    "delete": [
+      "anonymous"
+    ]
+  },
+  "owner": "anonymous",
+  "data": {
+    "title": "Microscopy of Cobalt Samples"
+  },
+  "meta": {},
+  "curators": [],
+  "created": 1669560885.988901,
+  "createdDate": "2022-11-27T09:54:45",
+  "lastModified": 1669560885.988901,
+  "lastModifiedDate": "2022-11-27T09:54:45",
+  "deactivated": null,
+  "type": "dmp"
+}
+```
+
+Clients should note the value of the `id` property in order to make further updates.
+
+### Updating a record's data content
+
+Typically in the life of project record, after the client creates a new record, it will incrementally
+update its data content as the user manipulates the client interface.  Updates can be made using either
+PUT or PATCH requests.  PATCH is perhaps more typical: the input can contain partial data that will be
+merged with the data that is already saved on the server.  With PUT, the input can also contain partial
+data; however, it will completely replace the data that was already saved on the server, deleting all
+previous data properties regardless of whether they are included in the input.
+
+In this example, we use PATCH to add more data to the record.  Note that the URL includes the record's
+`id` value, followed by `/data`.  
+
+```bash
+curl -X PATCH --data '{"expectedDataSize": "2 TB"}' http://localhost:9091/midas/dmp/mdm1/mdm1:0003/data
+```
+
+Because the URL used above specifically accesses the "data" part of the record, only the updated data
+object is returned:
+
+```
+{
+  "title": "Microscopy of Cobalt Samples",
+  "expectedDataSize": "2 TB"
+}
+```
+
+### Getting record contents (without updating)
+
+GET requests can be made to against different resource URLs to get full records or portions of a record.
+In particular:
+
+  - `http://localhost:9091/midas/dmp/mdm1` -- returns a list of records that the requesting user is
+    allowed to read
+  - `http://localhost:9091/midas/dmp/mdm1/`_id_ -- returns the full record that has an identifier given
+    by _id_.  The format is the same as that returned by the create request illustrated above.
+  - `http://localhost:9091/midas/dmp/mdm1/`_id_`/data` -- returns just the data contents for the record
+    with the identifier given by _id_.
+  - `http://localhost:9091/midas/dmp/mdm1/`_id_`/name` -- returns just the mnemonic name assigned to the
+    record by the user
+  - `http://localhost:9091/midas/dmp/mdm1/`_id_`/owner` -- returns just the identifier of the user that 
+    owns (and usually created) the record
+  - `http://localhost:9091/midas/dmp/mdm1/`_id_`/acls` -- returns just access control lists attached to 
+    the record
+  - `http://localhost:9091/midas/dmp/mdm1/`_id_`/meta` -- returns just the custom metadata attached to record
+
+### Other operations of note
+
+The service provides other operations that a client can provide support for:
+
+  - **Changing the mnemonic name** - a new name can be assigned to the record via a PUT request on the
+    `/name` sub-resource of a record.
+  - **Add/Remove permissions for other users** - the `/acls` sub-resource endpoints allow one to manipulate
+    permissions given to other users.
+  - **Create user groups**<sup>*</sup> -- the `/midas/groups` endpoint alls a user to create and manage 
+    their own user groups that can be applied to a records ACLs.
+
+For more information, consult the [API documentation](https://localhost:9091/docs/dmpsvc-elements.html).  
+
+<sup>*</sup>_Not implemented yet._
+
+

docker/midasserver/entrypoint.sh

@@ -0,0 +1,29 @@
+#! /bin/bash
+#
+port=9091
+script=/dev/oar-pdr-py/scripts/midas-uwsgi.py
+[ -f "$script" ] || script=/app/dist/pdr/bin/midas-uwsgi.py
+
+[ -n "$OAR_WORKING_DIR" ] || OAR_WORKING_DIR=`mktemp --tmpdir -d _midasserver.XXXXX`
+[ -d "$OAR_WORKING_DIR" ] || {
+    echo midasserver: ${OAR_WORKING_DIR}: working directory does not exist
+    exit 10
+}
+[ -n "$OAR_LOG_DIR" ] || export OAR_LOG_DIR=$OAR_WORKING_DIR
+[ -n "$OAR_MIDASSERVER_CONFIG" ] || OAR_MIDASSERVER_CONFIG=/app/midas-config.yml
+
+echo
+echo Working Dir: $OAR_WORKING_DIR
+echo Access the MIDAS web services at http://localhost:$port/
+echo
+
+opts=
+oar_midas_db_type=$1
+[ -z "$oar_midas_db_type" ] || opts="--set-ph oar_midas_db_type=$oar_midas_db_type"
+
+echo '++' uwsgi --plugin python3 --http-socket :$port --wsgi-file $script --static-map /docs=/docs \
+                --set-ph oar_config_file=$OAR_MIDASSERVER_CONFIG \
+                --set-ph oar_working_dir=$OAR_WORKING_DIR $opts
+uwsgi --plugin python3 --http-socket :$port --wsgi-file $script --static-map /docs=/docs \
+      --set-ph oar_config_file=$OAR_MIDASSERVER_CONFIG \
+      --set-ph oar_working_dir=$OAR_WORKING_DIR $opts

docker/midasserver/midas-dmp_conf.yml

@@ -0,0 +1,35 @@
+logfile: midas-dmp.log
+loglevel: DEBUG
+dbio:
+   factory:  fsbased
+about: 
+   title: "MIDAS Authoring Services"
+   describedBy: "http://localhost:9091/midas/docs"
+   href: "http://localhost:9091/midas/"
+services: 
+   dmp: 
+      about: 
+         message:     "DMP Service is available"
+         title:       "Data Management Plan (DMP) Authoring API"
+         describedBy: "http://localhost:9091/docs/dmpsvc-elements.html"
+         href:        "http://localhost:9091/midas/dmp"
+
+      clients: 
+          midas: 
+             default_shoulder: mdm1
+          default: 
+             default_shoulder: mdm1
+
+      dbio: 
+         superusers:                [ "rlp3" ]
+         allowed_project_shoulders: ["mdm0", "mdm1"]
+         default_shoulder:          mdm1
+
+      default_convention: mdm1
+      conventions: 
+         mdm1: 
+            about: 
+                title:       "Data Management Plan (DMP) Authoring API (mdm1 convention)"
+                describedBy: "http://localhost:9091/docs/dmpsvc-elements.html"
+                href:        "http://localhost:9091/midas/dmp/mdm1"
+                version:     mdm1