Merge pull request #13 from usnistgov/py10

Upgrade to Python 3.10 and MongoDB 7.0 (see PR# 13)

.gitignore

@@ -49,8 +49,11 @@ m2repo
 
 python/ejsonschema-master/
 python/ejsonschema.zip
+VERSION
 python/nistoar/midas/version.py
 python/nistoar/pdr/version.py
+python/nistoar/web/version.py
+python/nistoar/nsd/version.py
 python/pynoid-master/
 python/pynoid.zip
 python/multibag-py-0.3

docker/midasserver/mongo/mongo.env

@@ -4,5 +4,5 @@ OAR_MONGODB_ADMIN_PASS=admin
 OAR_MONGODB_USER=oarop
 OAR_MONGODB_PASS=oarop
 OAR_MONGODB_DBNAME=midas
-MONGO_VERSION=4.4.8
+MONGO_VERSION=7.0
 set +a

docker/midasserver/run.sh

@@ -46,6 +46,8 @@ ARGUMENTS
   -c FILE, --config-file FILE   Use a custom service configuration given in FILE.
                                 This file must be in YAML or JSON format.
                                 Defaut: docker/midasserver/midas-dmp_config.yml
+  -B, --bg                      Run the server in the background (returning the 
+                                command prompt after successful launch)
   -M, --use-mongodb             Use a MongoDB backend; DIR must also be provided.
                                 If not set, a file-based database (using JSON 
                                 files) will be used, stored under DIR/dbfiles.
@@ -77,6 +79,7 @@ CONFIGFILE=
 USEMONGO=
 STOREDIR=
 DBTYPE=
+DETACH=
 while [ "$1" != "" ]; do
     case "$1" in
         -b|--build)
@@ -99,6 +102,9 @@ while [ "$1" != "" ]; do
         --port=*)
             PORT=`echo $1 | sed -e 's/[^=]*=//'`
             ;;
+        -B|--bg|--detach)
+            DETACH="--detach"
+            ;;
         -M|--use-mongo)
             DBTYPE="mongo"
             ;;
@@ -227,7 +233,7 @@ if [ "$ACTION" = "stop" ]; then
     stop_server || true
     $STOP_MONGO
 else
-    echo '+' docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $PACKAGE_NAME/midasserver $DBTYPE
-    docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $PACKAGE_NAME/midasserver $DBTYPE
+    echo '+' docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $DETACH $PACKAGE_NAME/midasserver $DBTYPE
+    docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $DETACH $PACKAGE_NAME/midasserver $DBTYPE
 fi
 

docker/pdrpytest/Dockerfile

@@ -16,6 +16,8 @@ FROM oar-pdr-py/pyenv
 
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod a+rx /usr/local/bin/entrypoint.sh
+COPY runtests.sh /usr/local/bin/runtests.sh
+RUN chmod a+rx /usr/local/bin/runtests.sh
 
 VOLUME /dev/oar-pdr-py
 VOLUME /app/dist
@@ -24,7 +26,8 @@ RUN mkdir -p /dev/oar-pdr-py /app && chmod a+rwx /app
 WORKDIR /dev/oar-pdr-py
 
 ENV PYTHONPATH /dev/oar-pdr-py/python/build/lib
+ENV MONGO_TESTDB_URL mongodb://localhost/testdb
 
 ARG devuser=developer
-USER $devuser
+ENV DEV_USER $devuser
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker/pdrpytest/entrypoint.sh

@@ -1,68 +1,20 @@
 #! /bin/bash
 #
+mongod_ctl=/usr/local/bin/mongod_ctl.sh
 [ "$1" = "" ] && exec /bin/bash
 
-function install {
-    scripts/install.sh --prefix=/app/pdr || return 1
-    export OAR_HOME=/app/pdr
-    export PYTHONPATH=$OAR_HOME/lib/python
-    export OAR_LOG_DIR=$OAR_HOME/var/logs
-}
-
-function exitopwith { 
-    echo $2 > $1.exit
-    exit $2
-}
-
-cmd=$1
 case "$1" in
-    makedist)
-        shift
-        scripts/makedist "$@"
-        ;;
-    testall)
-        install || {
-            echo "testall: Failed to install oar-pdr-py"
-            exitopwith testall 2
-        }
-        shift
-        scripts/testall "$@"; stat=$?
-
-        [ "$stat" != "0" ] && {
-            echo "testall: One or more tests failed (last=$stat)"
-            exitopwith testall 3
-        }
-
-        echo All python tests passed
-        ;;
-    install)
-        install
-        python -c 'import nistoar.pdr, jq'
-        ;;
-    testshell)
-        # libdir=`ls /dev/oar-pdr-py/python/build | grep lib.`
-        export OAR_PYTHONPATH=/dev/oar-pdr-py/python/build/lib
-        export OAR_JQ_LIB=/dev/oar-pdr-py/metadata/jq
-        export OAR_MERGE_ETC=/dev/oar-pdr-py/metadata/etc/merge
-        export OAR_SCHEMA_DIR=/dev/oar-pdr-py/metadata/model
-        export PYTHONPATH=$OAR_PYTHONPATH
-        exec /bin/bash
-        ;;
     shell)
         exec /bin/bash
         ;;
-    installshell)
-        install
-        exec /bin/bash
+    testall|testshell)
+        [ -x $mongo_ctl ] && $mongod_ctl start && sleep 1
+        exec /usr/local/bin/gosu developer /usr/local/bin/runtests.sh $@
         ;;
     *)
-        echo Unknown command: $1
-        echo Available commands:  makedist testall testshell install shell installshell testmdservshell testpreserveshell
+        exec /usr/local/bin/gosu developer /usr/local/bin/runtests.sh $@
         ;;
 esac
 
-[ $? -ne 0 ] && exitopwith $cmd 1
-true
-
 
 

docker/pdrpytest/runtests.sh

@@ -0,0 +1,70 @@
+#! /bin/bash
+#
+function install {
+    scripts/setversion.sh
+    scripts/install.sh --prefix=/app/pdr || return 1
+    export OAR_HOME=/app/pdr
+    export PYTHONPATH=$OAR_HOME/lib/python
+    export OAR_LOG_DIR=$OAR_HOME/var/logs
+}
+
+function exitopwith { 
+    echo $2 > $1.exit
+    exit $2
+}
+
+
+
+cmd=$1
+case "$1" in
+    makedist)
+        shift
+        scripts/makedist "$@"
+        EXCODE=$?
+        ;;
+    build)
+        scripts/setversion.sh
+        (cd python && python setup.py build)
+        EXCODE=$?
+        ;;
+    testall)
+        install || {
+            echo "testall: Failed to install oar-pdr-py"
+            exitopwith testall 2
+        }
+        shift
+
+        # wrapper root shell should have already started mongodb
+        stat=0
+        scripts/testall "$@"; stat=$?
+
+        [ "$stat" != "0" ] && {
+            echo "testall: One or more test packages failed (last=$stat)"
+            echo NOT OK
+            exitopwith testall 3
+        }
+        # echo All OK
+        EXCODE=$stat
+        ;;
+    install)
+        install
+        python -c 'import nistoar.pdr, jq'
+        EXCODE=$?
+        ;;
+    testshell)
+        # wrapper root shell should have already started mongodb
+        install
+        exec /bin/bash
+        ;;
+    shell)
+        exec /bin/bash
+        ;;
+    *)
+        echo Unknown command: $1
+        echo Available commands:  build makedist testall install shell
+        EXCODE=100
+        ;;
+esac
+
+echo $EXCODE > $cmd.exit
+exit $EXCODE

docker/peopleserver/mongo/mongo.env

@@ -4,5 +4,5 @@ OAR_MONGODB_ADMIN_PASS=admin
 OAR_MONGODB_USER=oarop
 OAR_MONGODB_PASS=oarop
 OAR_MONGODB_DBNAME=midas
-MONGO_VERSION=4.4.8
+MONGO_VERSION=7.0
 set +a

docker/pyenv/Dockerfile

@@ -11,20 +11,20 @@
 #########################################################################
 FROM oar-metadata/ejsonschema
 
-RUN apt-get update && apt-get install -y python-yaml curl wget less sudo zip \
+RUN apt-get update && apt-get install -y python3-yaml curl wget less sudo zip \
                                          p7zip-full ca-certificates git
 # RUN pip install --upgrade pip setuptools
 RUN pip install funcsigs 'bagit>=1.6.3,<2.0' 'fs>=2.0.21' jsonpatch mako pyjwt \
                 jsonpath_ng lxml webdavclient3
 
 # install multibag from source
-RUN multibag_ver=0.3 && \
+RUN multibag_ver=0.5 && \
     curl -L -o multibag-py.zip \
     https://github.com/usnistgov/multibag-py/archive/$multibag_ver.zip && \
     unzip -oq multibag-py.zip && \
     cd multibag-py-$multibag_ver && \
     echo __version__ = $multibag_ver >> multibag/__init__.py && \
-    python setup.py install --install-purelib=/usr/local/lib/python3.8/dist-packages
+    python setup.py install --install-purelib=/usr/local/lib/python3.10/dist-packages
 
 # Create the user that container operations should run as.  Normally,
 # this is set to match identity information of the host user that is

python/nistoar/midas/dbio/mongo.py

@@ -226,10 +226,6 @@ def adv_select_records(self, filter: dict,
             filter["$and"].append(constraints)
             try:
                 coll = self.native[self._projcoll]
-                index_info = coll.list_indexes()
-                index_exists = any(idx['name'] == 'name_text_data.keywords_text' for idx in index_info)
-                if not index_exists:
-                    coll.create_index([("$**", "text")],weights= {"name": 2, "data.keywords": 3})
                 for rec in coll.find(filter, {'_id': False}):
                     yield base.ProjectRecord(self._projcoll, rec, self)
 
@@ -317,4 +313,4 @@ def __init__(self, config: Mapping, dburl: str = None):
 
     def create_client(self, servicetype: str, config: Mapping = {}, foruser: str = base.ANONYMOUS):
         cfg = merge_config(config, deepcopy(self._cfg))
-        return MongoDBClient(self._dburl, cfg, servicetype, foruser)
+        return MongoDBClient(self._dburl, cfg, servicetype, foruser)

python/nistoar/midas/dbio/wsgi/base.py

@@ -113,20 +113,33 @@ def get_json_body(self):
             else:
                 out = json.load(bodyin, object_pairs_hook=OrderedDict)
             if self._reqrec:
-                self._reqrec.add_body_text(json.dumps(name, indent=2)).record()
+                self._reqrec.add_body_text(json.dumps(out, indent=2)).record()
             return out
 
         except (ValueError, TypeError) as ex:
             if self.log.isEnabledFor(logging.DEBUG):
                 self.log.error("Failed to parse input: %s", str(ex))
                 self.log.debug("\n%s", body)
             if self._reqrec:
-                self._reqrec.add_body_text(body).record()
+                try:
+                    if isinstance(body, bytes):
+                        body = body.decode('utf-8')
+                    self._reqrec.add_body_text(body).record()
+                except Exception:
+                    self._reqrec.record()
             raise self.FatalError(400, "Input not parseable as JSON",
                                   "Input document is not parse-able as JSON: "+str(ex))
 
         except Exception as ex:
             if self._reqrec:
-                self._reqrec.add_body_text(body).record()
+                try:
+                    if body:
+                        if isinstance(body, bytes):
+                            body = body.decode('utf-8')
+                        self._reqrec.add_body_text(body).record()
+                    else:
+                        self._reqrec.add_body_text("<<empty body>>")
+                except Exception:
+                    self._reqrec.add_body_text("<<not convertable to text>>")
             raise
 

python/nistoar/pdr/health/servicechecker/__init__.py

@@ -2,7 +2,7 @@
 A module that can check the health of running services by sending test queries.  
 """
 import re, textwrap
-from collections import Sequence
+from collections.abc import Sequence
 
 import requests
 try:

python/nistoar/pdr/notify/base.py

@@ -24,7 +24,8 @@ class are specialized for a particular channel, and instances are
 """
 from datetime import datetime
 from abc import ABCMeta, abstractmethod, abstractproperty
-from collections import Mapping, OrderedDict
+from collections import OrderedDict
+from collections.abc import Mapping
 import json
 
 from ... import pdr as _pdr

python/nistoar/pdr/preserve/bagit/builder.py

@@ -5,7 +5,8 @@
 import pynoid as noid
 from shutil import copy as filecopy, rmtree
 from copy import deepcopy
-from collections import Mapping, Sequence, OrderedDict
+from collections import OrderedDict
+from collections.abc import Mapping, Sequence
 from urllib.parse import quote as urlencode
 
 from .. import PreservationSystem
@@ -243,7 +244,7 @@ def __init__(self, parentdir, bagname, config=None, id=None, logger=None):
                 self._nerdm_schema_id += '#'
 
     def __del__(self):
-        self.disconnect_logfile()
+        self.disconnect_logfile(quiet=True)
 
     def __enter__(self):
         return self
@@ -321,7 +322,7 @@ def connect_logfile(self, logfile=None, loglevel=NORM):
 
         self.plog.addHandler(hdlr)
 
-    def disconnect_logfile(self, logfile=None):
+    def disconnect_logfile(self, logfile=None, quiet=False):
         """
         disconnect the log file from this builder.  This ensures that the 
         logfile is closed so that the bag can be savely removed, moved, etc.
@@ -345,8 +346,9 @@ def disconnect_logfile(self, logfile=None):
                 logfile = os.path.join(self.bagdir, logfile)
             files = [ logfile ]
 
-        self.log.debug("Disconnecting BagBuilder from internal log (under %s)", self.bagdir)
-
+        if not quiet:
+            self.log.debug("Disconnecting BagBuilder from internal log (under %s)", self.bagdir)
+
         for lf in files:
             hdlrs = [h for h in self.plog.handlers if self._handles_logfile(h,lf)]
             for h in hdlrs:

python/nistoar/pdr/preserve/bagit/tools/enhance.py

@@ -3,7 +3,8 @@
 NIST-style bag.  
 """
 import re, logging
-from collections import OrderedDict, Mapping
+from collections import OrderedDict
+from collections.abc import Mapping
 
 from nistoar.nerdm.convert import DOIResolver
 from nistoar.doi import is_DOI, DOIResolutionException

python/nistoar/pdr/preserve/bagit/validate/base.py

@@ -2,7 +2,8 @@
 This module provides the base validator class
 """
 from abc import ABCMeta, abstractmethod, abstractproperty
-from collections import Sequence, OrderedDict
+from collections import OrderedDict
+from collections.abc import Sequence
 
 ERROR = 1
 WARN  = 2