Merge advance searching, Py3.10 from integration into main

.gitignore

@@ -8,6 +8,7 @@ __pycache__
 # Ignore .c files by default to avoid including generated code. If you want to
 # add a non-generated .c extension, use `git add -f filename.c`.
 *.c
+docker/dockbuild.log
 
 # Other generated files
 */version.py
@@ -46,6 +47,16 @@ m2repo
 .pydevproject
 .settings
 
+python/ejsonschema-master/
+python/ejsonschema.zip
+VERSION
+python/nistoar/midas/version.py
+python/nistoar/pdr/version.py
+python/nistoar/web/version.py
+python/nistoar/nsd/version.py
+python/pynoid-master/
+python/pynoid.zip
+python/multibag-py-0.3
 # Mac OSX
 .DS_Store
 

README.md

@@ -174,6 +174,14 @@ Similarly, `testall.docker` runs the tests in a container:
 Like their non-docker counterparts, these scripts accept product names
 as arguments.
 
+### MONGODB AND LOCAL TEST
+docker-compose up
+export PYTHONPATH=$PATH_TO_LIB
+export MONGO_TESTDB_URL="mongodb://admin:admin@127.0.0.1:27017/midas?authSource=admin"
+python3 setup.py build
+python3 test/nistoar/midas/dbio
+
+
 ## Running the services
 
 The [scripts](scripts) directory contains

docker/dockbuild.sh

@@ -24,7 +24,7 @@ PACKAGE_NAME=oar-pdr-py
 ## depends the former ones).  
 #
 DEP_DOCKER_IMAGE_DIRS="pymongo jqfromsrc ejsonschema pyenv"
-EXEC_DOCKER_IMAGE_DIRS="pdrpytest pdpserver midasserver"
+EXEC_DOCKER_IMAGE_DIRS="pdrpytest pdpserver midasserver peopleserver"
 
 [ -d "$codedir/metadata/docker" ] || {
     echo ${prog}: Missing metadata submodule
@@ -65,3 +65,7 @@ if { echo " $BUILD_IMAGES " | grep -qs " midasserver "; }; then
     echo '+' docker build $BUILD_OPTS -t $PACKAGE_NAME/midasserver midasserver | logit
     docker build $BUILD_OPTS -t $PACKAGE_NAME/midasserver midasserver 2>&1 | logit
 fi
+if { echo " $BUILD_IMAGES " | grep -qs " peopleserver "; }; then
+    echo '+' docker build $BUILD_OPTS -t $PACKAGE_NAME/peopleserver peopleserver | logit
+    docker build $BUILD_OPTS -t $PACKAGE_NAME/peopleserver peopleserver 2>&1 | logit
+fi

docker/midasserver/midas-dmpdap_conf.yml

@@ -6,7 +6,8 @@ about:
    title: "MIDAS Authoring Services"
    describedBy: "http://localhost:9091/midas/docs"
    href: "http://localhost:9091/midas/"
-jwt_auth:
+authentication:
+   type: jwt
    key: "tokensecret"
    require_expiration: false
 services: 

docker/midasserver/mongo/docker-compose.mongo.yml

@@ -10,8 +10,9 @@ services:
       - MONGO_INITDB_ROOT_USERNAME=${OAR_MONGODB_ADMIN_USER}
       - MONGO_INITDB_ROOT_PASSWORD=${OAR_MONGODB_ADMIN_PASS}
       - MONGO_INITDB_DATABASE=${OAR_MONGODB_DBNAME}
-      - MONGO_USER=${OAR_MONGODB_USER}
-      - MONGO_PASS=${OAR_MONGODB_PASS}
+      - OAR_MONGODB_USER=${OAR_MONGODB_USER}
+      - OAR_MONGODB_PASS=${OAR_MONGODB_PASS}
+      - OAR_MONGODB_DBNAME=${OAR_MONGODB_DBNAME}
     volumes:
       - ${OAR_MONGODB_DBDIR}:/data/db
       - ./mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh:ro
@@ -25,6 +26,8 @@ services:
     links:
       - mongodb
     environment:
+      ME_CONFIG_BASICAUTH_USERNAME:    ${OAR_MONGODB_ADMIN_USER}
+      ME_CONFIG_BASICAUTH_PASSWORD:    ${OAR_MONGODB_ADMIN_PASS}
       ME_CONFIG_MONGODB_ADMINUSERNAME: ${OAR_MONGODB_ADMIN_USER}
-      ME_CONFIG_MONGODB_ADMINUSERNAME: ${OAR_MONGODB_ADMIN_PASS}
+      ME_CONFIG_MONGODB_ADMINUSERPASS: ${OAR_MONGODB_ADMIN_PASS}
       ME_CONFIG_MONGODB_URL: mongodb://${OAR_MONGODB_ADMIN_USER}:${OAR_MONGODB_ADMIN_PASS}@mongodb:27017/

docker/midasserver/mongo/mongo-init.sh

@@ -1,4 +1,4 @@
-echo "Creating curator user..."
+echo "Creating MIDAS user..."
 echo '
     use '${OAR_MONGODB_DBNAME}'
     db.createUser(
@@ -8,4 +8,4 @@ echo '
             roles: [ "readWrite" ]
         }
     )
-    exit' | mongo
+    exit' | mongo -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD

docker/midasserver/mongo/mongo.env

@@ -4,5 +4,5 @@ OAR_MONGODB_ADMIN_PASS=admin
 OAR_MONGODB_USER=oarop
 OAR_MONGODB_PASS=oarop
 OAR_MONGODB_DBNAME=midas
-MONGO_VERSION=4.4.8
+MONGO_VERSION=7.0
 set +a

docker/midasserver/run.sh

@@ -46,9 +46,13 @@ ARGUMENTS
   -c FILE, --config-file FILE   Use a custom service configuration given in FILE.
                                 This file must be in YAML or JSON format.
                                 Defaut: docker/midasserver/midas-dmp_config.yml
+  -B, --bg                      Run the server in the background (returning the 
+                                command prompt after successful launch)
   -M, --use-mongodb             Use a MongoDB backend; DIR must also be provided.
                                 If not set, a file-based database (using JSON 
                                 files) will be used, stored under DIR/dbfiles.
+  -p, --port NUM                The port that the service should listen to 
+                                (default: 9091)
   -h, --help                    Print this text to the terminal and then exit
 
 EOF
@@ -68,12 +72,14 @@ function build_server_image {
     $dockerdir/dockbuild.sh -d midasserver # > log
 }
 
+PORT=9091
 DOPYBUILD=
 DODOCKBUILD=
 CONFIGFILE=
 USEMONGO=
 STOREDIR=
 DBTYPE=
+DETACH=
 while [ "$1" != "" ]; do
     case "$1" in
         -b|--build)
@@ -89,6 +95,16 @@ while [ "$1" != "" ]; do
         --config-file=*)
             CONFIGFILE=`echo $1 | sed -e 's/[^=]*=//'`
             ;;
+        -p)
+            shift
+            PORT=$1
+            ;;
+        --port=*)
+            PORT=`echo $1 | sed -e 's/[^=]*=//'`
+            ;;
+        -B|--bg|--detach)
+            DETACH="--detach"
+            ;;
         -M|--use-mongo)
             DBTYPE="mongo"
             ;;
@@ -217,7 +233,7 @@ if [ "$ACTION" = "stop" ]; then
     stop_server || true
     $STOP_MONGO
 else
-    echo '+' docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:9091:9091/tcp --rm --name=$CONTAINER_NAME $PACKAGE_NAME/midasserver $DBTYPE
-    docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:9091:9091/tcp --rm --name=$CONTAINER_NAME $PACKAGE_NAME/midasserver $DBTYPE
+    echo '+' docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $DETACH $PACKAGE_NAME/midasserver $DBTYPE
+    docker run $ENVOPTS $VOLOPTS $NETOPTS -p 127.0.0.1:${PORT}:${PORT}/tcp --rm --name=$CONTAINER_NAME $DETACH $PACKAGE_NAME/midasserver $DBTYPE
 fi
 

docker/pdpserver/pdr-pdp_config.yml

@@ -2,10 +2,10 @@ logfile: pdp.log
 authorized: 
   - auth_key: "NCNRTOKEN"
     user:     "gurn"
-    group:    "ncnr"
+    client:    "ncnr"
   - auth_key: "DRAFTTOKEN"
     user:     "draft"
-    group:    "test"
+    client:    "test"
 
 conventions: 
   pdp0:

docker/pdrpytest/Dockerfile

@@ -16,6 +16,8 @@ FROM oar-pdr-py/pyenv
 
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod a+rx /usr/local/bin/entrypoint.sh
+COPY runtests.sh /usr/local/bin/runtests.sh
+RUN chmod a+rx /usr/local/bin/runtests.sh
 
 VOLUME /dev/oar-pdr-py
 VOLUME /app/dist
@@ -24,7 +26,8 @@ RUN mkdir -p /dev/oar-pdr-py /app && chmod a+rwx /app
 WORKDIR /dev/oar-pdr-py
 
 ENV PYTHONPATH /dev/oar-pdr-py/python/build/lib
+ENV MONGO_TESTDB_URL mongodb://localhost/testdb
 
 ARG devuser=developer
-USER $devuser
+ENV DEV_USER $devuser
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker/pdrpytest/entrypoint.sh

@@ -1,68 +1,20 @@
 #! /bin/bash
 #
+mongod_ctl=/usr/local/bin/mongod_ctl.sh
 [ "$1" = "" ] && exec /bin/bash
 
-function install {
-    scripts/install.sh --prefix=/app/pdr || return 1
-    export OAR_HOME=/app/pdr
-    export PYTHONPATH=$OAR_HOME/lib/python
-    export OAR_LOG_DIR=$OAR_HOME/var/logs
-}
-
-function exitopwith { 
-    echo $2 > $1.exit
-    exit $2
-}
-
-cmd=$1
 case "$1" in
-    makedist)
-        shift
-        scripts/makedist "$@"
-        ;;
-    testall)
-        install || {
-            echo "testall: Failed to install oar-pdr-py"
-            exitopwith testall 2
-        }
-        shift
-        scripts/testall "$@"; stat=$?
-
-        [ "$stat" != "0" ] && {
-            echo "testall: One or more tests failed (last=$stat)"
-            exitopwith testall 3
-        }
-
-        echo All python tests passed
-        ;;
-    install)
-        install
-        python -c 'import nistoar.pdr, jq'
-        ;;
-    testshell)
-        # libdir=`ls /dev/oar-pdr-py/python/build | grep lib.`
-        export OAR_PYTHONPATH=/dev/oar-pdr-py/python/build/lib
-        export OAR_JQ_LIB=/dev/oar-pdr-py/metadata/jq
-        export OAR_MERGE_ETC=/dev/oar-pdr-py/metadata/etc/merge
-        export OAR_SCHEMA_DIR=/dev/oar-pdr-py/metadata/model
-        export PYTHONPATH=$OAR_PYTHONPATH
-        exec /bin/bash
-        ;;
     shell)
         exec /bin/bash
         ;;
-    installshell)
-        install
-        exec /bin/bash
+    testall|testshell)
+        [ -x $mongo_ctl ] && $mongod_ctl start && sleep 1
+        exec /usr/local/bin/gosu developer /usr/local/bin/runtests.sh $@
         ;;
     *)
-        echo Unknown command: $1
-        echo Available commands:  makedist testall testshell install shell installshell testmdservshell testpreserveshell
+        exec /usr/local/bin/gosu developer /usr/local/bin/runtests.sh $@
         ;;
 esac
 
-[ $? -ne 0 ] && exitopwith $cmd 1
-true
-
 
 

docker/pdrpytest/runtests.sh

@@ -0,0 +1,70 @@
+#! /bin/bash
+#
+function install {
+    scripts/setversion.sh
+    scripts/install.sh --prefix=/app/pdr || return 1
+    export OAR_HOME=/app/pdr
+    export PYTHONPATH=$OAR_HOME/lib/python
+    export OAR_LOG_DIR=$OAR_HOME/var/logs
+}
+
+function exitopwith { 
+    echo $2 > $1.exit
+    exit $2
+}
+
+
+
+cmd=$1
+case "$1" in
+    makedist)
+        shift
+        scripts/makedist "$@"
+        EXCODE=$?
+        ;;
+    build)
+        scripts/setversion.sh
+        (cd python && python setup.py build)
+        EXCODE=$?
+        ;;
+    testall)
+        install || {
+            echo "testall: Failed to install oar-pdr-py"
+            exitopwith testall 2
+        }
+        shift
+
+        # wrapper root shell should have already started mongodb
+        stat=0
+        scripts/testall "$@"; stat=$?
+
+        [ "$stat" != "0" ] && {
+            echo "testall: One or more test packages failed (last=$stat)"
+            echo NOT OK
+            exitopwith testall 3
+        }
+        # echo All OK
+        EXCODE=$stat
+        ;;
+    install)
+        install
+        python -c 'import nistoar.pdr, jq'
+        EXCODE=$?
+        ;;
+    testshell)
+        # wrapper root shell should have already started mongodb
+        install
+        exec /bin/bash
+        ;;
+    shell)
+        exec /bin/bash
+        ;;
+    *)
+        echo Unknown command: $1
+        echo Available commands:  build makedist testall install shell
+        EXCODE=100
+        ;;
+esac
+
+echo $EXCODE > $cmd.exit
+exit $EXCODE

docker/peopleserver/Dockerfile

@@ -0,0 +1,23 @@
+#########################################################################
+#
+# Programmatic Data Publishing (PDP) web service
+#
+# This container launches the PDP web service via scripts/pdp-uwsgi.py
+#
+#########################################################################
+FROM oar-pdr-py/pyenv
+
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod a+rx /usr/local/bin/entrypoint.sh
+
+VOLUME /dev/oar-pdr-py
+VOLUME /app/dist
+
+RUN mkdir -p /dev/oar-pdr-py /app /data/nsd && chmod a+rwx /app
+WORKDIR /dev/oar-pdr-py
+
+ENV PYTHONPATH /app/dist/pdr/lib/python
+
+ARG devuser=developer
+USER $devuser
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker/peopleserver/entrypoint.sh

@@ -0,0 +1,21 @@
+#! /bin/bash
+#
+script=scripts/people-uwsgi.py
+port=9092
+[ -z "$OAR_PEOPLESERVER_PORT" ] || port=$OAR_PEOPLESERVER_PORT
+[ -n "$OAR_WORKING_DIR" ] || OAR_WORKING_DIR=`mktemp --tmpdir -d _nsdserver.XXXXX`
+[ -d "$OAR_WORKING_DIR" ] || {
+    echo peopleserver: ${OAR_WORKING_DIR}: working directory does not exist
+    exit 10
+}
+[ -n "$OAR_LOG_DIR" ] || export OAR_LOG_DIR=$OAR_WORKING_DIR
+[ -n "$OAR_PEOPLESERVER_CONFIG" ] || OAR_PEOPLESERVER_CONFIG=docker/peopleserver/people_config.yml
+
+echo
+echo Working Dir: $OAR_WORKING_DIR
+echo Access the NSD web service at http://localhost:$port/
+echo
+
+uwsgi --plugin python3 --http-socket :$port --wsgi-file $script \
+      --set-ph oar_config_file=$OAR_PEOPLESERVER_CONFIG \
+      --set-ph oar_working_dir=$OAR_WORKING_DIR