Merge pull request #135 from qld-gov-au/QOLDEV-313-ckan-2.10

[QOLDEV-313] sync tests with other repos

.ahoy.yml

@@ -19,6 +19,7 @@ commands:
   build-network:
     usage: Ensure that the amazeeio network exists.
     cmd: |
+      ahoy title "Creating amazeeio Docker network"
       docker network prune -f > /dev/null
       docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
 
@@ -34,13 +35,14 @@ commands:
   up:
     usage: Build and start Docker containers.
     cmd: |
+      ahoy title "Building and starting Docker containers"
       docker-compose up -d "$@"
+      echo "Initialising database schema"
       ahoy cli '$APP_DIR/bin/init.sh'
-      sleep 10
-      docker-compose logs
+      echo "Waiting for containers to start listening..."
       ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
-      if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
-      if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
+      if docker-compose logs | grep -q "\[Error\]"; then exit 1; fi
+      if docker-compose logs | grep -q "Exception"; then exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
       export DOCTOR_CHECK_CLI=0
 
@@ -66,7 +68,7 @@ commands:
 
   pull:
     usage: Pull latest docker images.
-    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep ckan/ckan- | grep -v none | xargs -n1 docker pull | cat; fi
 
   cli:
     usage: Start a shell inside CLI container or run a command.
@@ -78,11 +80,14 @@ commands:
 
   install-site:
     usage: Install test site data.
-    cmd: ahoy cli '$APP_DIR/bin/create-test-data.sh'
+    cmd: |
+      ahoy title "Installing a fresh site"
+      ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh'
 
   clean:
     usage: Remove containers and all build files.
     cmd: |
+      ahoy title "Cleaning up old builds"
       ahoy down
       # Remove other directories.
       # @todo: Add destinations below.
@@ -104,6 +109,7 @@ commands:
   lint:
     usage: Lint code.
     cmd: |
+      ahoy title 'Check for lint'
       ahoy cli "flake8 ${@:-ckanext}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
@@ -112,7 +118,7 @@ commands:
     cmd: |
       docker cp . $(docker-compose ps -q ckan):/srv/app/
       docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
-      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test.ini $CKAN_INI'
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI; $APP_DIR/bin/process-config.sh'
 
   pip-list:
     usage: List pip install version details
@@ -125,17 +131,20 @@ commands:
       ahoy lint
       ahoy test-unit
       ahoy install-site
-      ahoy test-bdd || (ahoy logs; exit 1)
+      ahoy test-bdd
 
   test-unit:
     usage: Run unit tests.
     cmd: |
+      ahoy title 'Run unit tests'
       ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
     usage: Run BDD tests.
     cmd: |
+      ahoy title 'Run scenario tests'
+      ahoy cli "rm -f test/screenshots/*"
       ahoy start-mailmock &
       sleep 5 &&
       ahoy cli "behave -k ${*:-test/features}" --tags=smoke $BEHAVE_TAGS && \

.docker/Dockerfile-template.ckan

@@ -2,6 +2,7 @@ FROM openknowledge/ckan-dev:{CKAN_VERSION}
 
 ARG SITE_URL=http://ckan:5000/
 ENV PYTHON_VERSION={PYTHON_VERSION}
+ENV CKAN_VERSION={CKAN_VERSION}
 ENV CKAN_SITE_URL="${SITE_URL}"
 ENV PYTHON={PYTHON}
 

.github/workflows/test.yml

@@ -9,10 +9,10 @@ on:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.x'
       - name: Install requirements
@@ -28,7 +28,7 @@ jobs:
         ckan-version: ["2.10", 2.9, 2.9-py2]
         ckan-type: ['vanilla', 'custom']
 
-    name: Test on CKAN ${{ matrix.ckan-version }} ${{ matrix.ckan-type }}
+    name: CKAN ${{ matrix.ckan-version }} ${{ matrix.ckan-type }}
     runs-on: ubuntu-latest
     container: drevops/ci-builder
     env:
@@ -43,16 +43,14 @@ jobs:
         run: bin/build.sh
         timeout-minutes: 15
 
-      - name: Test
-        run: |
-          ahoy lint
-          ahoy test-unit
+      - name: Unit test
+        run: bin/test.sh
         timeout-minutes: 10
 
-      - name: Test BDD
+      - name: Scenario test
         run: |
-           ahoy install-site
-           ahoy test-bdd
+          ahoy install-site
+          ahoy test-bdd
         timeout-minutes: 30
 
       - name: Retrieve logs

bin/build.sh

@@ -20,6 +20,11 @@ CKAN_GIT_VERSION=$CKAN_VERSION
 CKAN_GIT_ORG=ckan
 
 if [ "$CKAN_VERSION" = "2.10" ]; then
+    if [ "$CKAN_TYPE" = "custom" ]; then
+      CKAN_GIT_VERSION=ckan-2.10.0-qgov.1
+      CKAN_GIT_ORG=qld-gov-au
+    fi
+
     PYTHON_VERSION=py3
     PYTHON="${PYTHON}3"
 else
@@ -44,4 +49,4 @@ sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
     | sed "s|{PYTHON}|$PYTHON|g" \
     > .docker/Dockerfile.ckan
 
-ahoy build || (ahoy logs; exit 1)
+ahoy build

bin/ckan_cli

@@ -59,14 +59,14 @@ fi
 if [ "$COMMAND" = "ckan" ]; then
     # adjust args to match ckan expectations
     COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/')
-    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2
     exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS
 elif [ "$COMMAND" = "paster" ]; then
     # adjust args to match paster expectations
     COMMAND=$1
-    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2
     if [ "$1" = "show" ]; then shift; fi
     exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI}
 else

bin/create-test-data.sh

@@ -2,7 +2,7 @@
 ##
 # Create some example content for extension BDD tests.
 #
-set -e
+set -ex
 
 CKAN_ACTION_URL=${CKAN_SITE_URL}api/action
 CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
@@ -45,7 +45,8 @@ echo "Creating ${TEST_ORG_TITLE} organisation:"
 
 TEST_ORG=$( \
     curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'"}' \
+    --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'",
+        "description": "Organisation for testing issues"}' \
     ${CKAN_ACTION_URL}/organization_create
 )
 
@@ -68,9 +69,6 @@ curl -LsH "Authorization: ${API_KEY}" \
 # END.
 #
 
-add_user_if_needed organisation_admin "Organisation Admin" organisation_admin@localhost
-add_user_if_needed editor "Publisher" publisher@localhost
-add_user_if_needed foodie "Foodie" foodie@localhost
 add_user_if_needed group_admin "Group Admin" group_admin@localhost
 add_user_if_needed walker "Walker" walker@localhost
 
@@ -110,42 +108,26 @@ organisation_admin_update=$( \
 )
 echo ${organisation_admin_update}
 
-echo "Updating publisher to have editor privileges in the department-of-health Organisation:"
-publisher_update=$( \
-    curl -LsH "Authorization: ${API_KEY}" \
-    --data "id=department-of-health&username=editor&role=editor" \
-    ${CKAN_ACTION_URL}/organization_member_create
-)
-echo ${publisher_update}
-
-echo "Updating foodie to have admin privileges in the food-standards-agency Organisation:"
-foodie_update=$( \
-    curl -LsH "Authorization: ${API_KEY}" \
-    --data "id=food-standards-agency&username=foodie&role=admin" \
-    ${CKAN_ACTION_URL}/organization_member_create
-)
-echo ${foodie_update}
-
 echo "Creating non-organisation group:"
 group_create=$( \
     curl -LsH "Authorization: ${API_KEY}" \
-    --data "name=silly-walks" \
+    --data '{"name": "silly-walks", "title": "Silly walks", "description": "The Ministry of Silly Walks"}' \
     ${CKAN_ACTION_URL}/group_create
 )
 echo ${group_create}
 
 echo "Updating group_admin to have admin privileges in the silly-walks group:"
 group_admin_update=$( \
     curl -LsH "Authorization: ${API_KEY}" \
-    --data "id=silly-walks&username=group_admin&role=admin" \
+    --data '{"id": "silly-walks", "username": "group_admin", "role": "admin"}' \
     ${CKAN_ACTION_URL}/group_member_create
 )
 echo ${group_admin_update}
 
 echo "Updating walker to have editor privileges in the silly-walks group:"
 walker_update=$( \
     curl -LsH "Authorization: ${API_KEY}" \
-    --data "id=silly-walks&username=walker&role=editor" \
+    --data '{"id": "silly-walks", "username": "walker", "role": "editor"}' \
     ${CKAN_ACTION_URL}/group_member_create
 )
 echo ${walker_update}

bin/init-ext.sh

@@ -2,14 +2,20 @@
 ##
 # Install current extension.
 #
-set -e
-set -x
+set -ex
 
 install_requirements () {
     PROJECT_DIR=$1
     shift
     # Identify the best match requirements file, ignore the others.
-    # If there is one specific to our Python version, use that.
+    # If there is one specific to our CKAN or Python version, use that.
+    for filename_pattern in "$@"; do
+        filename="$PROJECT_DIR/${filename_pattern}-$CKAN_VERSION.txt"
+        if [ -f "$filename" ]; then
+            pip install -r "$filename"
+            return 0
+        fi
+    done
     for filename_pattern in "$@"; do
         filename="$PROJECT_DIR/${filename_pattern}-$PYTHON_VERSION.txt"
         if [ -f "$filename" ]; then
@@ -38,7 +44,5 @@ installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1
 # Validate that the extension was installed correctly.
 if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi
 
-if [ -d "$SRC_DIR/ckan/ckanext/activity" ]; then
-    sed -i 's|^ckan.plugins = |ckan.plugins = activity |' $CKAN_INI
-fi
+. $APP_DIR/bin/process-config.sh
 . ${APP_DIR}/bin/deactivate

bin/init.sh

@@ -7,4 +7,4 @@ set -e
 . ${APP_DIR}/bin/activate
 CLICK_ARGS="--yes" ckan_cli db clean
 ckan_cli db init
-ckan_cli db upgrade
+

bin/process-config.sh

@@ -0,0 +1,5 @@
+if [ -d "$SRC_DIR/ckan/ckanext/activity" ]; then
+    sed -i 's|^ckan.plugins =|ckan.plugins = activity|' $CKAN_INI
+    sed -i 's|^ckan.plugins =|ckan.plugins = activity|' .docker/test.ini
+fi
+

bin/serve.sh

@@ -7,15 +7,17 @@ dockerize -wait tcp://redis:6379 -timeout 1m
 
 for i in `seq 1 60`; do
     if (PGPASSWORD=pass psql -h postgres -U ckan_default -d ckan_test -c "\q"); then
+        echo "Database became ready on attempt $i"
         break
     else
+        echo "Database not yet ready, retrying (attempt $i)..."
         sleep 1
     fi
 done
 
 . ${APP_DIR}/bin/activate
 if (which ckan > /dev/null); then
-    ckan -c ${CKAN_INI} run -r
+    ckan -c ${CKAN_INI} run --disable-reloader
 else
     paster serve ${CKAN_INI}
 fi

bin/test.sh

@@ -2,15 +2,11 @@
 ##
 # Run tests in CI.
 #
-set -e
+set -ex
 
-echo "==> Lint code"
 ahoy lint
 
-echo "==> Run Unit tests"
 ahoy test-unit
 
-echo "==> Run BDD tests"
 ahoy install-site
-ahoy cli "rm -r test/screenshots || true"
-ahoy test-bdd || (ahoy logs; exit 1)
+ahoy test-bdd

ckanext/qgov/common/controller.py

@@ -17,7 +17,7 @@
 from ckan import model
 from ckan.controllers.package import PackageController
 from ckan.lib.render import TemplateNotFound
-from ckan.plugins.toolkit import _, abort, asbool, config, g, get_action,\
+from ckan.plugins.toolkit import _, abort, asbool, config, g, get_action, \
     redirect_to, render, request, url_for, ObjectNotFound, NotAuthorized
 
 LOG = getLogger(__name__)

ckanext/qgov/common/plugin.py

@@ -14,7 +14,7 @@
 import ckan.lib.navl.dictization_functions as df
 from ckan.lib.navl.validators import unicode_safe
 from ckan.plugins import implements, SingletonPlugin
-from ckan.plugins.toolkit import _, add_template_directory, get_action,\
+from ckan.plugins.toolkit import _, add_template_directory, get_action, \
     get_validator, render
 
 from . import authenticator, auth_functions as auth, helpers, intercepts, urlm

ckanext/qgov/common/views/dataset.py

@@ -5,7 +5,7 @@
 from flask import Blueprint
 
 from ckan import model
-from ckan.plugins.toolkit import check_ckan_version, g, get_action,\
+from ckan.plugins.toolkit import check_ckan_version, g, get_action, \
     redirect_to, url_for, ObjectNotFound, NotAuthorized
 from ckan.views import dataset, resource
 

dev-requirements-2.9-py2.txt

@@ -0,0 +1,14 @@
+behave==1.2.6
+behaving==2.0.0
+Appium-Python-Client<=0.52
+ckanapi==4.3
+ckantoolkit>=0.0.4
+factory-boy
+faker==3.0.1
+pytest-factoryboy
+flake8==3.8.3
+mock
+pytest-ckan
+six>=1.13.0
+splinter>=0.13.0,<0.17
+

dev-requirements-2.9.txt

@@ -0,0 +1,12 @@
+behaving==2.0.0
+Appium-Python-Client==1.3.0
+ckanapi==4.3
+ckantoolkit>=0.0.4
+factory-boy
+Faker
+flake8==3.8.3
+mock
+pytest-ckan
+six>=1.13.0
+splinter>=0.13.0,<0.17
+