[QOLDEV-982] add testing of CKAN 2.11 and master

- Replace obsolete test containers with ones that support CKAN 2.11
- Sync test config with other repos

.ahoy.yml

@@ -38,7 +38,7 @@ commands:
       ahoy title "Building and starting Docker containers"
       sh bin/docker-compose.sh up -d "$@"
       echo "Initialising database schema"
-      ahoy cli '$APP_DIR/bin/init.sh'
+      ahoy cli '"${APP_DIR}"/bin/init.sh'
       echo "Waiting for containers to start listening..."
       ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
       if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
@@ -79,9 +79,9 @@ commands:
     cmd: |
       CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
       if [ "${#}" -ne 0 \]; then
-        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
+        docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate; cd $APP_DIR;'" $*"
       else
-        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
+        docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate && cd $APP_DIR && sh'
       fi
 
   doctor:
@@ -92,7 +92,7 @@ commands:
     usage: Install test site data.
     cmd: |
       ahoy title "Installing a fresh site"
-      ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh'
+      ahoy cli '"${APP_DIR}"/bin/init.sh && "${APP_DIR}"/bin/create-test-data.sh'
 
   clean:
     usage: Remove containers and all build files.
@@ -128,13 +128,13 @@ commands:
     cmd: |
       docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
       docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
-      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI; $APP_DIR/bin/process-config.sh'
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli "${APP_DIR}"/bin/*; cp -v .docker/test.ini $CKAN_INI; "${APP_DIR}"/bin/process-config.sh'
 
   test-unit:
     usage: Run unit tests.
     cmd: |
       ahoy title 'Run unit tests'
-      ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
+      ahoy cli 'pytest --ckan-ini=${CKAN_INI} --cov=ckanext "${APP_DIR}"/ckanext --junit-xml=test/junit/results.xml' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
@@ -144,13 +144,14 @@ commands:
       ahoy cli "rm -f test/screenshots/*"
       ahoy start-mailmock &
       sleep 5
+      JUNIT_OUTPUT="--junit --junit-directory=test/junit/"
       if [ "$BEHAVE_TAG" = "" ]; then
-        (ahoy cli "behave -k ${*:-test/features} --tags=smoke" \
-         && ahoy cli "behave -k ${*:-test/features}"
+        (ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=smoke" \
+         && ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=-smoke"
         ) || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       else
         # run tests with the specified tag
-        ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG" \
+        ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=$BEHAVE_TAG" \
           || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       fi
       ahoy stop-mailmock
@@ -159,7 +160,7 @@ commands:
     usage: Starts email mock server used for email BDD tests
     cmd: |
       ahoy title 'Starting mailmock'
-      ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout
+      ahoy cli 'mailmock -p 8025 -o "${APP_DIR}"/test/emails' # for debugging mailmock email output remove --no-stdout
 
   stop-mailmock:
     usage: Stops email mock server used for email BDD tests

.docker/Dockerfile-template.ckan

@@ -1,5 +1,9 @@
 FROM ckan/ckan-dev:{CKAN_VERSION}
 
+# swap between root and unprivileged user
+ARG ORIGINAL_USER
+RUN ORIGINAL_USER=$(id -un)
+
 ARG SITE_URL=http://ckan:5000/
 ENV PYTHON_VERSION={PYTHON_VERSION}
 ENV CKAN_VERSION={CKAN_VERSION}
@@ -8,12 +12,21 @@ ENV PYTHON={PYTHON}
 
 WORKDIR "${APP_DIR}"
 
-ENV DOCKERIZE_VERSION v0.6.1
+COPY .docker/test.ini $CKAN_INI
+
+COPY . "${APP_DIR}"/
+
+USER root
+
+COPY bin/ckan_cli /usr/bin/
+
+RUN chmod +x "${APP_DIR}"/bin/*.sh /usr/bin/ckan_cli
+
+ENV DOCKERIZE_VERSION=v0.6.1
 RUN wget -O - https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
         | tar -C /usr/local/bin -xzvf -
 
-# Update urllib3 to fix urlopen bug
-RUN pip install -U urllib3
+RUN which ps || apt-get install -y procps
 
 # Install CKAN.
 
@@ -24,15 +37,9 @@ RUN cd $SRC_DIR/ckan \
     && git reset --hard && git clean -f \
     && git checkout '{CKAN_GIT_VERSION}'
 
-COPY .docker/test.ini $CKAN_INI
-
-COPY . ${APP_DIR}/
-
-COPY bin/ckan_cli /usr/bin/
-
-RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
+USER "$ORIGINAL_USER"
 
 # Init current extension.
-RUN ${APP_DIR}/bin/init-ext.sh
+RUN "${APP_DIR}"/bin/init-ext.sh
 
 CMD ["/srv/app/bin/serve.sh"]

.docker/test.ini

@@ -1,16 +1,3 @@
-#
-# CKAN - Pylons configuration
-#
-# These are some of the configuration options available for your CKAN
-# instance. Check the documentation in 'doc/configuration.rst' or at the
-# following URL for a description of what they do and the full list of
-# available options:
-#
-# http://docs.ckan.org/en/latest/maintaining/configuration.html
-#
-# The %(here)s variable will be replaced with the parent directory of this file
-#
-
 [DEFAULT]
 debug = false
 smtp_server = localhost:8025
@@ -30,14 +17,9 @@ full_stack = true
 cache_dir = /tmp/%(ckan.site_id)s/
 beaker.session.key = ckan
 
-# This is the secret token that the beaker library uses to hash the cookie sent
-# to the client. `paster make-config` generates a unique value for this each
-# time it generates a config file.
 SECRET_KEY = bSmgPpaxg2M+ZRes3u1TXwIcE
 beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE
 
-# `paster make-config` generates a unique value for this each time it generates
-# a config file.
 app_instance_uuid = 6e3daf8e-1c6b-443b-911f-c7ab4c5f9605
 
 # repoze.who config
@@ -68,6 +50,7 @@ ckan.auth.create_user_via_api = false
 ckan.auth.create_user_via_web = true
 ckan.auth.roles_that_cascade_to_sub_groups = admin
 ckan.auth.public_user_details = False
+ckan.auth.reveal_private_datasets = True
 
 
 ## Search Settings

.github/workflows/test.yml

@@ -27,6 +27,12 @@ jobs:
       fail-fast: false
       matrix:
         ckan-version: ["2.10", 2.9]
+        experimental: [false]
+        include:
+          - ckan-version: '2.11'
+            experimental: true  # Need to wait for 2.11.1 to fix org page
+          - ckan-version: 'master'
+            experimental: true  #master is unstable, good to know if we are compatible or not
 
     name: Test on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
@@ -40,37 +46,54 @@ jobs:
         run: echo HOME=/root >> "$GITHUB_ENV"
 
       - uses: actions/checkout@v4
+        continue-on-error: ${{ matrix.experimental }}
         timeout-minutes: 2
 
       - name: Build
+        continue-on-error: ${{ matrix.experimental }}
         run: bin/build.sh
         timeout-minutes: 15
 
-      - name: Test lint
-        run: bin/test-lint.sh
-        timeout-minutes: 30
+      - name: Lint
+        continue-on-error: ${{ matrix.experimental }}
+        run: ahoy lint
+        timeout-minutes: 10
 
-      - name: Test
-        run: bin/test.sh
-        timeout-minutes: 30
+      - name: Unit test
+        continue-on-error: ${{ matrix.experimental }}
+        run: ahoy test-unit
+        timeout-minutes: 10
 
-      - name: Test BDD
-        run: bin/test-bdd.sh
+      - name: Scenario test
+        continue-on-error: ${{ matrix.experimental }}
+        run: |
+          ahoy install-site
+          ahoy test-bdd
         timeout-minutes: 40
 
       - name: Retrieve logs
         if: always()
-        run: bin/get-logs.sh
+        run: ahoy logs
+        continue-on-error: ${{ matrix.experimental }}
         timeout-minutes: 1
 
-      - name: Retrieve screenshots
-        if: failure()
+      - name: Retrieve results
+        if: always()
         run: bin/process-artifacts.sh
+        continue-on-error: ${{ matrix.experimental }}
         timeout-minutes: 1
 
+      - name: Test Summary
+        uses: test-summary/action@v2
+        continue-on-error: ${{ matrix.experimental }}
+        with:
+          paths: "/tmp/artifacts/junit/*.xml"
+        if: always()
+
       - name: Upload screenshots
-        if: failure()
+        if: always()
         uses: actions/upload-artifact@v4
+        continue-on-error: ${{ matrix.experimental }}
         with:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots

bin/build.sh

@@ -23,9 +23,11 @@ CKAN_GIT_ORG=qld-gov-au
 if [ "$CKAN_VERSION" = "2.11" ]; then
     CKAN_GIT_VERSION=ckan-2.11.0
 elif [ "$CKAN_VERSION" = "2.10" ]; then
-    CKAN_GIT_VERSION=ckan-2.10.5-qgov.1
-else
+    CKAN_GIT_VERSION=ckan-2.10.5-qgov.4
+elif [ "$CKAN_VERSION" = "2.9" ]; then
     CKAN_GIT_VERSION=ckan-2.9.9-qgov.4
+elif [ "$CKAN_VERSION" = "master" ]; then
+    CKAN_GIT_ORG=ckan
 fi
 
 sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \

bin/create-test-data.sh

@@ -9,7 +9,7 @@ CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
 CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
 CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
 
-. ${APP_DIR}/bin/activate
+. "${APP_DIR}"/bin/activate
 
 add_user_if_needed () {
     echo "Adding user '$2' ($1) with email address [$3]"
@@ -19,6 +19,10 @@ add_user_if_needed () {
         password="${4:-Password123!}"
 }
 
+api_call () {
+    wget -O - --header="Authorization: ${API_KEY}" --post-data "$1" ${CKAN_ACTION_URL}/$2
+}
+
 add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL"
 ckan_cli sysadmin add "${CKAN_USER_NAME}"
 
@@ -33,9 +37,7 @@ fi
 #
 echo "Adding sysadmin config:"
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"ckanext.data_qld.excluded_display_name_words": "gov"}' \
-    ${CKAN_ACTION_URL}/config_option_update
+api_call '{"ckanext.data_qld.excluded_display_name_words": "gov"}' config_option_update
 
 ##
 # END.
@@ -56,16 +58,12 @@ add_user_if_needed test_org_member "Test Member" test_org_member@localhost
 
 echo "Creating ${TEST_ORG_TITLE} organisation:"
 
-api_call () {
-    wget -O - --header="Authorization: ${API_KEY}" --post-data "$1" ${CKAN_ACTION_URL}/$2
-}
-
 TEST_ORG=$( \
     api_call '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'",
         "description": "Organisation for testing issues"}' organization_create
 )
 
-TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py)
+TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON "${APP_DIR}"/bin/extract-id.py)
 
 echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):"
 

bin/docker-compose.sh

@@ -1,13 +1,15 @@
 #!/bin/sh
 
+set -x
+
 # Pass commands to Docker Compose v1 or v2 depending on what is present
 
-if (which docker-compose >/dev/null); then
-    # Docker Compose v1
-    docker-compose $*
-elif (docker compose ls >/dev/null); then
+if (docker compose ls >/dev/null); then
     # Docker Compose v2
     docker compose $*
+elif (which docker-compose >/dev/null); then
+    # Docker Compose v1
+    docker-compose $*
 else
     # Docker Compose not found
     exit 1

bin/init-ext.sh

@@ -32,9 +32,10 @@ install_requirements () {
     done
 }
 
-. ${APP_DIR}/bin/activate
-
-pip install --upgrade pip
+. "${APP_DIR}"/bin/activate
+if [ "$CKAN_VERSION" = "2.9" ]; then
+    pip install "setuptools>=44.1.0,<71"
+fi
 install_requirements . dev-requirements requirements-dev
 for extension in . `ls -d $SRC_DIR/ckanext-*`; do
     install_requirements $extension requirements pip-requirements
@@ -45,5 +46,5 @@ installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1
 # Validate that the extension was installed correctly.
 if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi
 
-. $APP_DIR/bin/process-config.sh
-. ${APP_DIR}/bin/deactivate
+. "${APP_DIR}"/bin/process-config.sh
+. "${APP_DIR}"/bin/deactivate

bin/init.sh

@@ -4,6 +4,6 @@
 #
 set -e
 
-. ${APP_DIR}/bin/activate
+. "${APP_DIR}"/bin/activate
 CLICK_ARGS="--yes" ckan_cli db clean
 ckan_cli db init

bin/process-artifacts.sh

@@ -6,8 +6,9 @@ set -e
 
 # Create screenshots directory in case it was not created before. This is to
 # avoid this script to fail when copying artifacts.
-ahoy cli "mkdir -p test/screenshots"
+ahoy cli "mkdir -p test/screenshots test/junit"
 
 # Copy from the app container to the build host for storage.
-mkdir -p /tmp/artifacts/behave
+mkdir -p /tmp/artifacts/behave /tmp/artifacts/junit
 docker cp "$(sh bin/docker-compose.sh ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/
+docker cp "$(sh bin/docker-compose.sh ps -q ckan)":/srv/app/test/junit /tmp/artifacts/

bin/serve.sh

@@ -1,21 +1,7 @@
 #!/usr/bin/env sh
 set -e
 
-dockerize -wait tcp://postgres:5432 -timeout 1m
-dockerize -wait tcp://solr:8983 -timeout 1m
-dockerize -wait tcp://redis:6379 -timeout 1m
-
-for i in `seq 1 60`; do
-    if (PGPASSWORD=pass psql -h postgres -U ckan_default -d ckan_test -c "\q"); then
-        echo "Database became ready on attempt $i"
-        break
-    else
-        echo "Database not yet ready, retrying (attempt $i)..."
-        sleep 1
-    fi
-done
-
-. ${APP_DIR}/bin/activate
+. "${APP_DIR}"/bin/activate
 if (which ckan > /dev/null); then
     ckan -c ${CKAN_INI} run --disable-reloader --threaded
 else