From 41a935995353cfd9496c99ad5ac6f68f6323d147 Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Sat, 22 Jun 2024 18:02:45 -0400
Subject: [PATCH 01/12] python 3.10: fix collection.abc imports

---
 python/nistoar/pdr/health/servicechecker/__init__.py |  2 +-
 python/nistoar/pdr/notify/base.py                    |  3 ++-
 python/nistoar/pdr/preserve/bagit/builder.py         | 12 +++++++-----
 python/nistoar/pdr/preserve/bagit/tools/enhance.py   |  3 ++-
 python/nistoar/pdr/preserve/bagit/validate/base.py   |  3 ++-
 python/nistoar/pdr/preserve/bagit/validate/nist.py   |  3 ++-
 python/nistoar/pdr/public/resolve/wsgi.py            |  3 ++-
 python/nistoar/pdr/publish/bagger/base.py            |  3 ++-
 python/nistoar/pdr/publish/bagger/datachecker.py     |  2 +-
 python/nistoar/pdr/publish/bagger/pdp.py             |  3 ++-
 python/nistoar/pdr/publish/bagger/prepupd.py         |  3 ++-
 python/nistoar/pdr/publish/bagger/utils.py           |  3 ++-
 python/nistoar/pdr/publish/idmint/minter.py          |  2 +-
 python/nistoar/pdr/publish/idmint/registry.py        |  3 ++-
 python/nistoar/pdr/publish/service/base.py           |  2 +-
 python/nistoar/pdr/publish/service/wsgi/__init__.py  |  3 ++-
 python/nistoar/pdr/utils/datamgmt.py                 |  2 +-
 python/nistoar/pdr/utils/io.py                       |  3 ++-
 python/tests/nistoar/pdr/utils/test_webrecord.py     |  3 ++-
 19 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/python/nistoar/pdr/health/servicechecker/__init__.py b/python/nistoar/pdr/health/servicechecker/__init__.py
index 618c2e4f..e94ec1b5 100644
--- a/python/nistoar/pdr/health/servicechecker/__init__.py
+++ b/python/nistoar/pdr/health/servicechecker/__init__.py
@@ -2,7 +2,7 @@
 A module that can check the health of running services by sending test queries.  
 """
 import re, textwrap
-from collections import Sequence
+from collections.abc import Sequence
 
 import requests
 try:
diff --git a/python/nistoar/pdr/notify/base.py b/python/nistoar/pdr/notify/base.py
index 37532bbc..2bd33b92 100644
--- a/python/nistoar/pdr/notify/base.py
+++ b/python/nistoar/pdr/notify/base.py
@@ -24,7 +24,8 @@ class are specialized for a particular channel, and instances are
 """
 from datetime import datetime
 from abc import ABCMeta, abstractmethod, abstractproperty
-from collections import Mapping, OrderedDict
+from collections import OrderedDict
+from collections.abc import Mapping
 import json
 
 from ... import pdr as _pdr
diff --git a/python/nistoar/pdr/preserve/bagit/builder.py b/python/nistoar/pdr/preserve/bagit/builder.py
index 67e65a59..144151fc 100644
--- a/python/nistoar/pdr/preserve/bagit/builder.py
+++ b/python/nistoar/pdr/preserve/bagit/builder.py
@@ -5,7 +5,8 @@
 import pynoid as noid
 from shutil import copy as filecopy, rmtree
 from copy import deepcopy
-from collections import Mapping, Sequence, OrderedDict
+from collections import OrderedDict
+from collections.abc import Mapping, Sequence
 from urllib.parse import quote as urlencode
 
 from .. import PreservationSystem
@@ -243,7 +244,7 @@ def __init__(self, parentdir, bagname, config=None, id=None, logger=None):
                 self._nerdm_schema_id += '#'
 
     def __del__(self):
-        self.disconnect_logfile()
+        self.disconnect_logfile(quiet=True)
 
     def __enter__(self):
         return self
@@ -321,7 +322,7 @@ def connect_logfile(self, logfile=None, loglevel=NORM):
         
         self.plog.addHandler(hdlr)
 
-    def disconnect_logfile(self, logfile=None):
+    def disconnect_logfile(self, logfile=None, quiet=False):
         """
         disconnect the log file from this builder.  This ensures that the 
         logfile is closed so that the bag can be savely removed, moved, etc.
@@ -345,8 +346,9 @@ def disconnect_logfile(self, logfile=None):
                 logfile = os.path.join(self.bagdir, logfile)
             files = [ logfile ]
 
-        self.log.debug("Disconnecting BagBuilder from internal log (under %s)", self.bagdir)
-
+        if not quiet:
+            self.log.debug("Disconnecting BagBuilder from internal log (under %s)", self.bagdir)
+        
         for lf in files:
             hdlrs = [h for h in self.plog.handlers if self._handles_logfile(h,lf)]
             for h in hdlrs:
diff --git a/python/nistoar/pdr/preserve/bagit/tools/enhance.py b/python/nistoar/pdr/preserve/bagit/tools/enhance.py
index 1b2372a1..4c008120 100644
--- a/python/nistoar/pdr/preserve/bagit/tools/enhance.py
+++ b/python/nistoar/pdr/preserve/bagit/tools/enhance.py
@@ -3,7 +3,8 @@
 NIST-style bag.  
 """
 import re, logging
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 
 from nistoar.nerdm.convert import DOIResolver
 from nistoar.doi import is_DOI, DOIResolutionException
diff --git a/python/nistoar/pdr/preserve/bagit/validate/base.py b/python/nistoar/pdr/preserve/bagit/validate/base.py
index fb6e53b4..e12a70c1 100644
--- a/python/nistoar/pdr/preserve/bagit/validate/base.py
+++ b/python/nistoar/pdr/preserve/bagit/validate/base.py
@@ -2,7 +2,8 @@
 This module provides the base validator class
 """
 from abc import ABCMeta, abstractmethod, abstractproperty
-from collections import Sequence, OrderedDict
+from collections import OrderedDict
+from collections.abc import Sequence
 
 ERROR = 1
 WARN  = 2
diff --git a/python/nistoar/pdr/preserve/bagit/validate/nist.py b/python/nistoar/pdr/preserve/bagit/validate/nist.py
index 03f3f2d7..b4993f60 100644
--- a/python/nistoar/pdr/preserve/bagit/validate/nist.py
+++ b/python/nistoar/pdr/preserve/bagit/validate/nist.py
@@ -2,7 +2,8 @@
 This module implements a validator for the NIST-generated bags
 """
 import os, re, json
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 from urllib.parse import urlparse
 
 import ejsonschema as ejs
diff --git a/python/nistoar/pdr/public/resolve/wsgi.py b/python/nistoar/pdr/public/resolve/wsgi.py
index d42cda4a..5e788116 100644
--- a/python/nistoar/pdr/public/resolve/wsgi.py
+++ b/python/nistoar/pdr/public/resolve/wsgi.py
@@ -3,7 +3,8 @@
 """
 import os, sys, logging, json, re
 from wsgiref.headers import Headers
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 from copy import deepcopy
 
 from ... import ARK_NAAN
diff --git a/python/nistoar/pdr/publish/bagger/base.py b/python/nistoar/pdr/publish/bagger/base.py
index 88c46822..4066fa17 100644
--- a/python/nistoar/pdr/publish/bagger/base.py
+++ b/python/nistoar/pdr/publish/bagger/base.py
@@ -4,7 +4,8 @@
 abstract base for subclasses that understand different input sources.
 """
 import os, json, filelock
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 from abc import ABCMeta, abstractmethod, abstractproperty
 from copy import deepcopy
 
diff --git a/python/nistoar/pdr/publish/bagger/datachecker.py b/python/nistoar/pdr/publish/bagger/datachecker.py
index 84784d0e..3438cea0 100644
--- a/python/nistoar/pdr/publish/bagger/datachecker.py
+++ b/python/nistoar/pdr/publish/bagger/datachecker.py
@@ -2,7 +2,7 @@
 tools for checking the availability of distributions described in a NIST bag.
 """
 import os, re
-from collections import Mapping
+from collections.abc import Mapping
 
 import multibag as mb
 import requests
diff --git a/python/nistoar/pdr/publish/bagger/pdp.py b/python/nistoar/pdr/publish/bagger/pdp.py
index f8de2544..e9f72966 100644
--- a/python/nistoar/pdr/publish/bagger/pdp.py
+++ b/python/nistoar/pdr/publish/bagger/pdp.py
@@ -3,7 +3,8 @@
 Publishing (PDP) API.  In this framework, SIP inputs are primarily in the form of NERDm metadata.
 """
 import os, re, logging, json
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 from abc import abstractmethod, abstractproperty
 from copy import deepcopy
 from urllib.parse import urlparse
diff --git a/python/nistoar/pdr/publish/bagger/prepupd.py b/python/nistoar/pdr/publish/bagger/prepupd.py
index e98e9107..2f4fb51d 100644
--- a/python/nistoar/pdr/publish/bagger/prepupd.py
+++ b/python/nistoar/pdr/publish/bagger/prepupd.py
@@ -22,7 +22,8 @@
 """
 import os, shutil, json, logging, re
 from abc import ABCMeta, abstractmethod, abstractproperty
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 from zipfile import ZipFile
 from time import mktime
 
diff --git a/python/nistoar/pdr/publish/bagger/utils.py b/python/nistoar/pdr/publish/bagger/utils.py
index 3e7b965c..83ea4e67 100644
--- a/python/nistoar/pdr/publish/bagger/utils.py
+++ b/python/nistoar/pdr/publish/bagger/utils.py
@@ -10,7 +10,8 @@
 repository).  
 """
 import os, re
-from collections import Sequence, Mapping, OrderedDict
+from collections import OrderedDict
+from collections.abc import Sequence, Mapping
 from urllib.parse import urlparse
 from copy import deepcopy
 
diff --git a/python/nistoar/pdr/publish/idmint/minter.py b/python/nistoar/pdr/publish/idmint/minter.py
index c7378291..a8401ec0 100644
--- a/python/nistoar/pdr/publish/idmint/minter.py
+++ b/python/nistoar/pdr/publish/idmint/minter.py
@@ -2,7 +2,7 @@
 Module providing minters for use with the PDR
 """
 import os, re
-from collections import Mapping
+from collections.abc import Mapping
 from copy import deepcopy
 from abc import abstractmethod
 
diff --git a/python/nistoar/pdr/publish/idmint/registry.py b/python/nistoar/pdr/publish/idmint/registry.py
index 3ab1538d..16594f86 100644
--- a/python/nistoar/pdr/publish/idmint/registry.py
+++ b/python/nistoar/pdr/publish/idmint/registry.py
@@ -2,7 +2,8 @@
 An implementation of an IDRegistry that tuned for use with the PDR
 """
 import os, logging, json, threading
-from collections import OrderedDict, Mapping, ChainMap
+from collections import OrderedDict
+from typing import Mapping, ChainMap
 from abc import ABCMeta, abstractmethod
 from typing import Callable
 from copy import deepcopy
diff --git a/python/nistoar/pdr/publish/service/base.py b/python/nistoar/pdr/publish/service/base.py
index 6eac1124..8cd2b179 100644
--- a/python/nistoar/pdr/publish/service/base.py
+++ b/python/nistoar/pdr/publish/service/base.py
@@ -3,7 +3,7 @@
 """
 import re, logging
 from abc import ABCMeta, abstractmethod, abstractproperty
-from collections import Mapping
+from collections.abc import Mapping
 from ...utils.prov import Agent
 from .. import PublishSystem, PublishingStateException, ConfigurationException
 from ....nerdm.constants import core_schema_base as NERDM_SCHEMA_BASE, CORE_SCHEMA_URI
diff --git a/python/nistoar/pdr/publish/service/wsgi/__init__.py b/python/nistoar/pdr/publish/service/wsgi/__init__.py
index 435ae544..72bbebaa 100644
--- a/python/nistoar/pdr/publish/service/wsgi/__init__.py
+++ b/python/nistoar/pdr/publish/service/wsgi/__init__.py
@@ -3,7 +3,8 @@
 """
 import os, sys, logging, json, re
 from wsgiref.headers import Headers
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 from copy import deepcopy
 from typing import List
 
diff --git a/python/nistoar/pdr/utils/datamgmt.py b/python/nistoar/pdr/utils/datamgmt.py
index ae00e475..541f6d22 100644
--- a/python/nistoar/pdr/utils/datamgmt.py
+++ b/python/nistoar/pdr/utils/datamgmt.py
@@ -1,7 +1,7 @@
 """
 Utility functions managing data and files
 """
-from collections import Mapping
+from collections.abc import Mapping
 import hashlib, os, shutil, time, re, math
 
 __all__ = [
diff --git a/python/nistoar/pdr/utils/io.py b/python/nistoar/pdr/utils/io.py
index 8b73d214..ff032559 100644
--- a/python/nistoar/pdr/utils/io.py
+++ b/python/nistoar/pdr/utils/io.py
@@ -1,7 +1,8 @@
 """
 Utility functions and classes for file reading and writing
 """
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 import json, os, threading
 try:
     import fcntl
diff --git a/python/tests/nistoar/pdr/utils/test_webrecord.py b/python/tests/nistoar/pdr/utils/test_webrecord.py
index 166e9132..40696062 100644
--- a/python/tests/nistoar/pdr/utils/test_webrecord.py
+++ b/python/tests/nistoar/pdr/utils/test_webrecord.py
@@ -1,6 +1,7 @@
 
 import os, pdb, requests, logging, time, json
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 from io import StringIO
 import unittest as test
 from copy import deepcopy

From de80133b1880b06e367efb838b9f4c3d812a66cf Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Sat, 22 Jun 2024 23:56:15 -0400
Subject: [PATCH 02/12] environment/ci upgrade:    * pull in updates in
 oar-metadata    * update pdr-specific dependencies    * launch mongo server
 for unit-testing

---
 docker/pdrpytest/Dockerfile    |  5 ++-
 docker/pdrpytest/entrypoint.sh | 58 +++--------------------------
 docker/pdrpytest/runtests.sh   | 67 ++++++++++++++++++++++++++++++++++
 docker/pyenv/Dockerfile        |  6 +--
 metadata                       |  2 +-
 5 files changed, 80 insertions(+), 58 deletions(-)
 create mode 100644 docker/pdrpytest/runtests.sh

diff --git a/docker/pdrpytest/Dockerfile b/docker/pdrpytest/Dockerfile
index 299141a6..9f836012 100644
--- a/docker/pdrpytest/Dockerfile
+++ b/docker/pdrpytest/Dockerfile
@@ -16,6 +16,8 @@ FROM oar-pdr-py/pyenv
 
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod a+rx /usr/local/bin/entrypoint.sh
+COPY runtests.sh /usr/local/bin/runtests.sh
+RUN chmod a+rx /usr/local/bin/runtests.sh
 
 VOLUME /dev/oar-pdr-py
 VOLUME /app/dist
@@ -24,7 +26,8 @@ RUN mkdir -p /dev/oar-pdr-py /app && chmod a+rwx /app
 WORKDIR /dev/oar-pdr-py
 
 ENV PYTHONPATH /dev/oar-pdr-py/python/build/lib
+ENV MONGO_TESTDB_URL mongodb://localhost/testdb
 
 ARG devuser=developer
-USER $devuser
+ENV DEV_USER $devuser
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/docker/pdrpytest/entrypoint.sh b/docker/pdrpytest/entrypoint.sh
index 4ac30820..99df66e8 100755
--- a/docker/pdrpytest/entrypoint.sh
+++ b/docker/pdrpytest/entrypoint.sh
@@ -1,68 +1,20 @@
 #! /bin/bash
 #
+mongod_ctl=/usr/local/bin/mongod_ctl.sh
 [ "$1" = "" ] && exec /bin/bash
 
-function install {
-    scripts/install.sh --prefix=/app/pdr || return 1
-    export OAR_HOME=/app/pdr
-    export PYTHONPATH=$OAR_HOME/lib/python
-    export OAR_LOG_DIR=$OAR_HOME/var/logs
-}
-
-function exitopwith { 
-    echo $2 > $1.exit
-    exit $2
-}
-
-cmd=$1
 case "$1" in
-    makedist)
-        shift
-        scripts/makedist "$@"
-        ;;
-    testall)
-        install || {
-            echo "testall: Failed to install oar-pdr-py"
-            exitopwith testall 2
-        }
-        shift
-        scripts/testall "$@"; stat=$?
-
-        [ "$stat" != "0" ] && {
-            echo "testall: One or more tests failed (last=$stat)"
-            exitopwith testall 3
-        }
-
-        echo All python tests passed
-        ;;
-    install)
-        install
-        python -c 'import nistoar.pdr, jq'
-        ;;
-    testshell)
-        # libdir=`ls /dev/oar-pdr-py/python/build | grep lib.`
-        export OAR_PYTHONPATH=/dev/oar-pdr-py/python/build/lib
-        export OAR_JQ_LIB=/dev/oar-pdr-py/metadata/jq
-        export OAR_MERGE_ETC=/dev/oar-pdr-py/metadata/etc/merge
-        export OAR_SCHEMA_DIR=/dev/oar-pdr-py/metadata/model
-        export PYTHONPATH=$OAR_PYTHONPATH
-        exec /bin/bash
-        ;;
     shell)
         exec /bin/bash
         ;;
-    installshell)
-        install
-        exec /bin/bash
+    testall|testshell)
+        [ -x $mongo_ctl ] && $mongod_ctl start && sleep 1
+        exec /usr/local/bin/gosu developer /usr/local/bin/runtests.sh $@
         ;;
     *)
-        echo Unknown command: $1
-        echo Available commands:  makedist testall testshell install shell installshell testmdservshell testpreserveshell
+        exec /usr/local/bin/gosu developer /usr/local/bin/mdtests.sh $@
         ;;
 esac
 
-[ $? -ne 0 ] && exitopwith $cmd 1
-true
-
     
     
diff --git a/docker/pdrpytest/runtests.sh b/docker/pdrpytest/runtests.sh
new file mode 100644
index 00000000..4869d80e
--- /dev/null
+++ b/docker/pdrpytest/runtests.sh
@@ -0,0 +1,67 @@
+#! /bin/bash
+#
+function install {
+    scripts/setversion.sh
+    scripts/install.sh --prefix=/app/pdr || return 1
+    export OAR_HOME=/app/pdr
+    export PYTHONPATH=$OAR_HOME/lib/python
+    export OAR_LOG_DIR=$OAR_HOME/var/logs
+}
+
+function exitopwith { 
+    echo $2 > $1.exit
+    exit $2
+}
+
+
+
+cmd=$1
+case "$1" in
+    makedist)
+        shift
+        scripts/makedist "$@"
+        ;;
+    build)
+        scripts/setversion.sh
+        (cd python && python setup.py build)
+        ;;
+    testall)
+        install || {
+            echo "testall: Failed to install oar-pdr-py"
+            exitopwith testall 2
+        }
+        shift
+
+        # wrapper root shell should have already started mongodb
+        stat=0
+        scripts/testall "$@"; stat=$?
+
+        [ "$stat" != "0" ] && {
+            echo "testall: One or more test packages failed (last=$stat)"
+            echo NOT OK
+            exitopwith testall 3
+        }
+        # echo All OK
+        ;;
+    install)
+        install
+        python -c 'import nistoar.pdr, jq'
+        ;;
+    testshell)
+        # wrapper root shell should have already started mongodb
+        install
+        exec /bin/bash
+        ;;
+    shell)
+        exec /bin/bash
+        ;;
+    *)
+        echo Unknown command: $1
+        echo Available commands:  build makedist testall install shell
+        exit 100
+        ;;
+esac
+
+EXCODE=$?
+echo $EXCODE > $cmd.exit
+exit $EXCODE
diff --git a/docker/pyenv/Dockerfile b/docker/pyenv/Dockerfile
index 4e954286..051dbc07 100644
--- a/docker/pyenv/Dockerfile
+++ b/docker/pyenv/Dockerfile
@@ -11,20 +11,20 @@
 #########################################################################
 FROM oar-metadata/ejsonschema
 
-RUN apt-get update && apt-get install -y python-yaml curl wget less sudo zip \
+RUN apt-get update && apt-get install -y python3-yaml curl wget less sudo zip \
                                          p7zip-full ca-certificates git
 # RUN pip install --upgrade pip setuptools
 RUN pip install funcsigs 'bagit>=1.6.3,<2.0' 'fs>=2.0.21' jsonpatch mako pyjwt \
                 jsonpath_ng lxml webdavclient3
 
 # install multibag from source
-RUN multibag_ver=0.3 && \
+RUN multibag_ver=0.5 && \
     curl -L -o multibag-py.zip \
     https://github.com/usnistgov/multibag-py/archive/$multibag_ver.zip && \
     unzip -oq multibag-py.zip && \
     cd multibag-py-$multibag_ver && \
     echo __version__ = $multibag_ver >> multibag/__init__.py && \
-    python setup.py install --install-purelib=/usr/local/lib/python3.8/dist-packages
+    python setup.py install --install-purelib=/usr/local/lib/python3.10/dist-packages
 
 # Create the user that container operations should run as.  Normally,
 # this is set to match identity information of the host user that is
diff --git a/metadata b/metadata
index 8fab4287..5e89601b 160000
--- a/metadata
+++ b/metadata
@@ -1 +1 @@
-Subproject commit 8fab42873a2c005e307bf21754f8dfae73d8f27a
+Subproject commit 5e89601b68598694955717ff934269b7a0321b02

From 1c45ce9c7d779c287432a6a5952935c23864f112 Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Sat, 22 Jun 2024 23:59:05 -0400
Subject: [PATCH 03/12] docker/run.sh: add --bg

---
 docker/midasserver/run.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/docker/midasserver/run.sh b/docker/midasserver/run.sh
index df6b7675..b5635aad 100755
--- a/docker/midasserver/run.sh
+++ b/docker/midasserver/run.sh
@@ -46,6 +46,8 @@ ARGUMENTS
   -c FILE, --config-file FILE   Use a custom service configuration given in FILE.
                                 This file must be in YAML or JSON format.
                                 Defaut: docker/midasserver/midas-dmp_config.yml
+  -B, --bg                      Run the server in the background (returning the 
+                                command prompt after successful launch)
   -M, --use-mongodb             Use a MongoDB backend; DIR must also be provided.
                                 If not set, a file-based database (using JSON 
                                 files) will be used, stored under DIR/dbfiles.
@@ -77,6 +79,7 @@ CONFIGFILE=
 USEMONGO=
 STOREDIR=
 DBTYPE=
+DETACH=
 while [ "$1" != "" ]; do
     case "$1" in
         -b|--build)
@@ -99,6 +102,9 @@ while [ "$1" != "" ]; do
         --port=*)
             PORT=`echo $1 | sed -e 's/[^=]*=//'`
             ;;
+        -B|--bg|--detach)
+            DETACH="--detach"
+            ;;
         -M|--use-mongo)
             DBTYPE="mongo"
             ;;
@@ -227,7 +233,7 @@ if [ "$ACTION" = "stop" ]; then
     stop_server || true
     $STOP_MONGO
 else
-    echo '+' docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $PACKAGE_NAME/midasserver $DBTYPE
-    docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $PACKAGE_NAME/midasserver $DBTYPE
+    echo '+' docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $DETACH $PACKAGE_NAME/midasserver $DBTYPE
+    docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $DETACH $PACKAGE_NAME/midasserver $DBTYPE
 fi
 

From 60ffcc3d4fd621312d52503c97a55a9f16ee4deb Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Sun, 23 Jun 2024 00:16:14 -0400
Subject: [PATCH 04/12] runtests.sh: fix exit code so that ci testing passes
 when tests pass

---
 docker/pdrpytest/runtests.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/docker/pdrpytest/runtests.sh b/docker/pdrpytest/runtests.sh
index 4869d80e..5eec6d10 100644
--- a/docker/pdrpytest/runtests.sh
+++ b/docker/pdrpytest/runtests.sh
@@ -20,10 +20,12 @@ case "$1" in
     makedist)
         shift
         scripts/makedist "$@"
+        EXCODE=$?
         ;;
     build)
         scripts/setversion.sh
         (cd python && python setup.py build)
+        EXCODE=$?
         ;;
     testall)
         install || {
@@ -42,10 +44,12 @@ case "$1" in
             exitopwith testall 3
         }
         # echo All OK
+        EXCODE=$stat
         ;;
     install)
         install
         python -c 'import nistoar.pdr, jq'
+        EXCODE=$?
         ;;
     testshell)
         # wrapper root shell should have already started mongodb
@@ -58,10 +62,9 @@ case "$1" in
     *)
         echo Unknown command: $1
         echo Available commands:  build makedist testall install shell
-        exit 100
+        EXCODE=100
         ;;
 esac
 
-EXCODE=$?
 echo $EXCODE > $cmd.exit
 exit $EXCODE

From c103cfc2243a2a48b4c8466e6b36b6216a33a205 Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Sun, 23 Jun 2024 08:32:38 -0400
Subject: [PATCH 05/12] update mongo version in standalone midasserver and
 peoplerserver

---
 docker/midasserver/mongo/mongo.env  | 2 +-
 docker/peopleserver/mongo/mongo.env | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/midasserver/mongo/mongo.env b/docker/midasserver/mongo/mongo.env
index 5993cd5f..4f9aefa1 100644
--- a/docker/midasserver/mongo/mongo.env
+++ b/docker/midasserver/mongo/mongo.env
@@ -4,5 +4,5 @@ OAR_MONGODB_ADMIN_PASS=admin
 OAR_MONGODB_USER=oarop
 OAR_MONGODB_PASS=oarop
 OAR_MONGODB_DBNAME=midas
-MONGO_VERSION=4.4.8
+MONGO_VERSION=7.0
 set +a
diff --git a/docker/peopleserver/mongo/mongo.env b/docker/peopleserver/mongo/mongo.env
index 5993cd5f..4f9aefa1 100644
--- a/docker/peopleserver/mongo/mongo.env
+++ b/docker/peopleserver/mongo/mongo.env
@@ -4,5 +4,5 @@ OAR_MONGODB_ADMIN_PASS=admin
 OAR_MONGODB_USER=oarop
 OAR_MONGODB_PASS=oarop
 OAR_MONGODB_DBNAME=midas
-MONGO_VERSION=4.4.8
+MONGO_VERSION=7.0
 set +a

From 6b7ace8ab47fd7e02c68dd437575e9109caebd34 Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Sun, 23 Jun 2024 09:54:41 -0400
Subject: [PATCH 06/12] env upgrade: fix pdrpytest/entrypoint.sh c/p typo

---
 docker/pdrpytest/entrypoint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/pdrpytest/entrypoint.sh b/docker/pdrpytest/entrypoint.sh
index 99df66e8..5f2b47fc 100755
--- a/docker/pdrpytest/entrypoint.sh
+++ b/docker/pdrpytest/entrypoint.sh
@@ -12,7 +12,7 @@ case "$1" in
         exec /usr/local/bin/gosu developer /usr/local/bin/runtests.sh $@
         ;;
     *)
-        exec /usr/local/bin/gosu developer /usr/local/bin/mdtests.sh $@
+        exec /usr/local/bin/gosu developer /usr/local/bin/runtests.sh $@
         ;;
 esac
 

From d90810a6c103895bd046ce50180a7099c00f9bea Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Mon, 24 Jun 2024 13:22:02 -0400
Subject: [PATCH 07/12] fix webrecord bugs in midas service

---
 python/nistoar/midas/dbio/wsgi/base.py | 19 ++++++++++++++++---
 python/nistoar/web/rest/base.py        |  1 +
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/python/nistoar/midas/dbio/wsgi/base.py b/python/nistoar/midas/dbio/wsgi/base.py
index 9d62222d..7fbf1399 100644
--- a/python/nistoar/midas/dbio/wsgi/base.py
+++ b/python/nistoar/midas/dbio/wsgi/base.py
@@ -113,7 +113,7 @@ def get_json_body(self):
             else:
                 out = json.load(bodyin, object_pairs_hook=OrderedDict)
             if self._reqrec:
-                self._reqrec.add_body_text(json.dumps(name, indent=2)).record()
+                self._reqrec.add_body_text(json.dumps(out, indent=2)).record()
             return out
 
         except (ValueError, TypeError) as ex:
@@ -121,12 +121,25 @@ def get_json_body(self):
                 self.log.error("Failed to parse input: %s", str(ex))
                 self.log.debug("\n%s", body)
             if self._reqrec:
-                self._reqrec.add_body_text(body).record()
+                try:
+                    if isinstance(body, bytes):
+                        body = body.decode('utf-8')
+                    self._reqrec.add_body_text(body).record()
+                except Exception:
+                    self._reqrec.record()
             raise self.FatalError(400, "Input not parseable as JSON",
                                   "Input document is not parse-able as JSON: "+str(ex))
 
         except Exception as ex:
             if self._reqrec:
-                self._reqrec.add_body_text(body).record()
+                try:
+                    if body:
+                        if isinstance(body, bytes):
+                            body = body.decode('utf-8')
+                        self._reqrec.add_body_text(body).record()
+                    else:
+                        self._reqrec.add_body_text("<<empty body>>")
+                except Exception:
+                    self._reqrec.add_body_text("<<not convertable to text>>")
             raise
 
diff --git a/python/nistoar/web/rest/base.py b/python/nistoar/web/rest/base.py
index c2604d5b..150d6479 100644
--- a/python/nistoar/web/rest/base.py
+++ b/python/nistoar/web/rest/base.py
@@ -19,6 +19,7 @@
 from ..webrecord import WebRecorder
 from ..formats import Unacceptable, UnsupportedFormat, FormatSupport
 from nistoar.base.config import ConfigurationException
+import nistoar.base.config as cfgmod
 from nistoar.pdr.utils.prov import Agent
 
 __all__ = ["Handler", "ServiceApp", "Unauthenticated", "WSGIApp", "AuthenticatedWSGIApp",

From 209bc4566b83674494760765d81ff41865b2c60c Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Mon, 24 Jun 2024 13:24:21 -0400
Subject: [PATCH 08/12] dbio.mongo: remove index-generating code (existence
 check not working)

---
 python/nistoar/midas/dbio/mongo.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/python/nistoar/midas/dbio/mongo.py b/python/nistoar/midas/dbio/mongo.py
index 8e9b5017..25193691 100644
--- a/python/nistoar/midas/dbio/mongo.py
+++ b/python/nistoar/midas/dbio/mongo.py
@@ -226,10 +226,6 @@ def adv_select_records(self, filter: dict,
             filter["$and"].append(constraints)
             try:
                 coll = self.native[self._projcoll]
-                index_info = coll.list_indexes()
-                index_exists = any(idx['name'] == 'name_text_data.keywords_text' for idx in index_info)
-                if not index_exists:
-                    coll.create_index([("$**", "text")],weights= {"name": 2, "data.keywords": 3})
                 for rec in coll.find(filter, {'_id': False}):
                     yield base.ProjectRecord(self._projcoll, rec, self)
 
@@ -317,4 +313,4 @@ def __init__(self, config: Mapping, dburl: str = None):
 
     def create_client(self, servicetype: str, config: Mapping = {}, foruser: str = base.ANONYMOUS):
         cfg = merge_config(config, deepcopy(self._cfg))
-        return MongoDBClient(self._dburl, cfg, servicetype, foruser)
\ No newline at end of file
+        return MongoDBClient(self._dburl, cfg, servicetype, foruser)

From 75c1d5ba208f5dc24fca5c2e64915d4e121bb0c2 Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Mon, 24 Jun 2024 14:07:14 -0400
Subject: [PATCH 09/12] test_mongo.py: create text index for
 adv_select_records() test

---
 python/tests/nistoar/midas/dbio/test_mongo.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/tests/nistoar/midas/dbio/test_mongo.py b/python/tests/nistoar/midas/dbio/test_mongo.py
index e6febf11..285a5945 100644
--- a/python/tests/nistoar/midas/dbio/test_mongo.py
+++ b/python/tests/nistoar/midas/dbio/test_mongo.py
@@ -322,6 +322,8 @@ def test_select_records(self):
 
     def test_adv_select_records(self):
 
+        self.cli.native[base.DMP_PROJECTS].create_index([("$**", "text")], weights={"name": 2})
+
         # inject some data into the database
         id = "pdr0:0002"
         rec = base.ProjectRecord(

From 25a5da307f336e49aa5bcd1b97e931f3d0c91eef Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Mon, 24 Jun 2024 15:37:39 -0400
Subject: [PATCH 10/12] web.rest: webrecord bug fix: import os

---
 python/nistoar/web/rest/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/nistoar/web/rest/base.py b/python/nistoar/web/rest/base.py
index 150d6479..c1839cbd 100644
--- a/python/nistoar/web/rest/base.py
+++ b/python/nistoar/web/rest/base.py
@@ -1,7 +1,7 @@
 """
 The base REST framework classes
 """
-import sys, re, json
+import sys, os, re, json
 from abc import ABCMeta, abstractmethod, abstractproperty
 from typing import Callable, List
 from functools import reduce

From 2e553dc41c0e805cf7c24bc7919bb8fa0564ce5d Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Mon, 24 Jun 2024 15:52:26 -0400
Subject: [PATCH 11/12] remove generated files checked in in error

---
 VERSION                       | 1 -
 python/nistoar/nsd/version.py | 6 ------
 python/nistoar/web/version.py | 6 ------
 3 files changed, 13 deletions(-)
 delete mode 100644 VERSION
 delete mode 100644 python/nistoar/nsd/version.py
 delete mode 100644 python/nistoar/web/version.py

diff --git a/VERSION b/VERSION
deleted file mode 100644
index ed507dba..00000000
--- a/VERSION
+++ /dev/null
@@ -1 +0,0 @@
-oar-pdr-py 2.1.0rc3
diff --git a/python/nistoar/nsd/version.py b/python/nistoar/nsd/version.py
deleted file mode 100644
index 445387c5..00000000
--- a/python/nistoar/nsd/version.py
+++ /dev/null
@@ -1,6 +0,0 @@
-"""
-An identification of the subsystem version.  Note that this module file gets 
-(over-) written by the build process.  
-"""
-
-__version__ = "2.1.0rc3"
diff --git a/python/nistoar/web/version.py b/python/nistoar/web/version.py
deleted file mode 100644
index 445387c5..00000000
--- a/python/nistoar/web/version.py
+++ /dev/null
@@ -1,6 +0,0 @@
-"""
-An identification of the subsystem version.  Note that this module file gets 
-(over-) written by the build process.  
-"""
-
-__version__ = "2.1.0rc3"

From 13066b8459972c24cad25afe113b7c6d55a88e52 Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Mon, 24 Jun 2024 15:56:00 -0400
Subject: [PATCH 12/12] .gitignore: add some additional generated files

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index e88186be..1d97bc44 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,8 +49,11 @@ m2repo
 
 python/ejsonschema-master/
 python/ejsonschema.zip
+VERSION
 python/nistoar/midas/version.py
 python/nistoar/pdr/version.py
+python/nistoar/web/version.py
+python/nistoar/nsd/version.py
 python/pynoid-master/
 python/pynoid.zip
 python/multibag-py-0.3