[QOLDEV-1005] update for CKAN 2.11

- update test container to support CKAN 2.11
- update to Selenium 4
- drop Python 2 support
- add Docker Compose 2 support
- sync test config with other repos

.ahoy.yml

@@ -20,6 +20,7 @@ commands:
   build-network:
     usage: Ensure that the amazeeio network exists.
     cmd: |
+      ahoy title "Creating amazeeio Docker network"
       docker network prune -f > /dev/null
       docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
 
@@ -28,63 +29,74 @@ commands:
     cmd: |
       ahoy line "Project                  : " ${PROJECT}
       ahoy line "Site local URL           : " ${LAGOON_LOCALDEV_URL}
-      ahoy line "DB port on host          : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
-      ahoy line "Solr port on host        : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+      ahoy line "DB port on host          : " $(docker port $(sh bin/docker-compose.sh ps -q postgres) 5432 | cut -d : -f 2)
+      ahoy line "Solr port on host        : " $(docker port $(sh bin/docker-compose.sh ps -q solr) 8983 | cut -d : -f 2)
       ahoy line "Mailhog URL              : " http://mailhog.docker.amazee.io/
 
   up:
     usage: Build and start Docker containers.
     cmd: |
-      docker-compose up -d "$@"
+      ahoy title "Building and starting Docker containers"
+      sh bin/docker-compose.sh up -d "$@"
       sleep 10
-      docker-compose logs
       ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
-      if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
-      if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "Exception"; then exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
       export DOCTOR_CHECK_CLI=0
 
   down:
     usage: Stop Docker containers and remove container, images, volumes and networks.
-    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
+    cmd: |
+      ahoy title 'Stopping and removing old containers, images, volumes, networks'
+      if [ -f "docker-compose.yml" ]; then sh bin/docker-compose.sh down --volumes; fi
 
   start:
     usage: Start existing Docker containers.
-    cmd: docker-compose start "$@"
+    cmd: sh bin/docker-compose.sh start "$@"
 
   stop:
     usage: Stop running Docker containers.
-    cmd: docker-compose stop "$@"
+    cmd: sh bin/docker-compose.sh stop "$@"
 
   restart:
     usage: Restart all stopped and running Docker containers.
-    cmd: docker-compose restart "$@"
+    cmd: sh bin/docker-compose.sh restart "$@"
 
   logs:
     usage: Show Docker logs.
-    cmd: docker-compose logs "$@"
+    cmd: |
+        ahoy title "Output logs"
+        sh bin/docker-compose.sh logs "$@"
 
   pull:
     usage: Pull latest docker images.
-    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep ckan/ckan- | grep -v none | xargs -n1 docker pull | cat; fi
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi
+    cmd: |
+      CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
+      if [ "${#}" -ne 0 \]; then
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
+      else
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
+      fi
 
   doctor:
     usage: Find problems with current project setup.
     cmd: bin/doctor.sh "$@"
 
   install-site:
-    usage: Install a site.
+    usage: Install test site data.
     cmd: |
       ahoy title "Installing a fresh site"
       ahoy cli '$APP_DIR/bin/init.sh'
 
   clean:
     usage: Remove containers and all build files.
     cmd: |
+      ahoy title "Cleaning up old builds"
       ahoy down
       # Remove other directories.
       # @todo: Add destinations below.
@@ -101,33 +113,37 @@ commands:
 
   flush-redis:
     usage: Flush Redis cache.
-    cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+    cmd: docker exec -i $(sh bin/docker-compose.sh ps -q redis) redis-cli flushall > /dev/null
 
   lint:
     usage: Lint code.
     cmd: |
+      ahoy title 'Check for lint'
       ahoy cli "flake8 ${@:-ckanext}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
   copy-local-files:
     usage: Update files from local repo.
     cmd: |
-      docker cp . $(docker-compose ps -q ckan):/srv/app/
-      docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
-      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test.ini $CKAN_INI'
+      docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
+      docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI'
 
   test-unit:
     usage: Run unit tests.
     cmd: |
+      ahoy title 'Run unit tests'
       ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
     usage: Run BDD tests.
     cmd: |
+      ahoy title 'Run scenario tests'
+      ahoy cli "rm -f test/screenshots/*"
       ahoy start-ckan-job-worker &
       ahoy start-mailmock &
-      sleep 5 &&
+      sleep 5
       ahoy cli "behave -k ${*:-test/features} --tags=-format_autocomplete" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       ahoy stop-mailmock
@@ -146,9 +162,9 @@ commands:
       ahoy cli "killall -2 mailmock"
 
   start-ckan-job-worker:
-    usage: Starts CKAN background job worker
+    usage: Starts default CKAN background job worker
     cmd: |
-      ahoy title 'Starting CKAN background job worker'
+      ahoy title 'Starting default CKAN background job worker'
       ahoy cli "ckan_cli jobs clear && \
                 ckan_cli jobs worker"
 

.docker/Dockerfile-template.ckan

@@ -1,19 +1,17 @@
-FROM openknowledge/ckan-dev:{CKAN_VERSION}
+FROM ckan/ckan-dev:{CKAN_VERSION}
 
 ARG SITE_URL=http://ckan:5000/
 ENV PYTHON_VERSION={PYTHON_VERSION}
+ENV CKAN_VERSION={CKAN_VERSION}
 ENV CKAN_SITE_URL="${SITE_URL}"
 ENV PYTHON={PYTHON}
 
 WORKDIR "${APP_DIR}"
 
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk add --no-cache build-base \
-    && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+RUN wget -O - https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
         | tar -C /usr/local/bin -xzvf -
 
-# Install CKAN.
-
 COPY .docker/test.ini $CKAN_INI
 
 COPY . ${APP_DIR}/

.env

@@ -13,7 +13,7 @@
 PROJECT="ckanext-report"
 
 # Docker Compose project name. All containers will have this name.
-COMPOSE_PROJECT_NAME="$PROJECT"
+COMPOSE_PROJECT_NAME="ckanext-report"
 
 # Flag to allow code linting failures. 0=enforce, 1=ignore
 ALLOW_LINT_FAIL=0

.github/workflows/test.yml

@@ -7,20 +7,39 @@ on:
       - master
 
 jobs:
+  # Quick check so we don't waste minutes if there's a Flake8 error
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Install requirements
+        run: pip install flake8 pycodestyle
+      - name: Check syntax
+        run: flake8 . --count --show-source --statistics
+
   test:
+    needs: lint
     strategy:
       fail-fast: false
       matrix:
-        ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7]
+        ckan-version: ["2.11", "2.10", 2.9]
 
-    name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
+    name: Test on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
-    container: integratedexperts/ci-builder
+    container: drevops/ci-runner:23.12.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
 
     steps:
-      - uses: actions/checkout@v2
+      # Patch https://github.com/actions/runner/issues/863
+      - name: Preserve $HOME set in the container
+        run: echo HOME=/root >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
         timeout-minutes: 2
 
       - name: Build
@@ -29,16 +48,21 @@ jobs:
 
       - name: Test
         run: bin/test.sh
-        timeout-minutes: 30
+        timeout-minutes: 15
+
+      - name: Retrieve logs
+        if: always()
+        run: ahoy logs
+        timeout-minutes: 1
 
       - name: Retrieve screenshots
-        if: failure()
+        if: always()
         run: bin/process-artifacts.sh
         timeout-minutes: 1
 
       - name: Upload screenshots
         if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots

README.md

@@ -25,10 +25,8 @@ TODO:
 
 | CKAN version    | Compatibility       |
 | --------------- | ------------------- |
-| 2.6 and earlier | yes                 |
-| 2.7             | yes                 |
-| 2.8             | yes                 |
-| 2.9-py2         | yes                 |
+| 2.8 and earlier | no                  |
+| 2.9-py2         | no                  |
 | 2.9  (py3)      | yes                 |
 | 2.10 (py3)      | yes                 |
 

bin/build.sh

@@ -2,7 +2,7 @@
 ##
 # Build site in CI.
 #
-set -e
+set -ex
 
 # Process Docker Compose configuration. This is used to avoid multiple
 # docker-compose.yml files.
@@ -14,16 +14,12 @@ sed -i -e "s/##//" docker-compose.yml
 # Pull the latest images.
 ahoy pull
 
-PYTHON=python
-if [ "$CKAN_VERSION" = "2.7" ] || [ "$CKAN_VERSION" = "2.8" ] || [ "$CKAN_VERSION" = "2.9-py2" ]; then
-    PYTHON_VERSION=py2
-else
-    PYTHON_VERSION=py3
-    PYTHON="${PYTHON}3"
-fi
+PYTHON_VERSION=py3
+PYTHON="python3"
 
 sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
+    | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" \
     | sed "s|{PYTHON}|$PYTHON|g" \
-    | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" > .docker/Dockerfile.ckan
+    > .docker/Dockerfile.ckan
 
-ahoy build || (ahoy logs; exit 1)
+ahoy build

bin/ckan_cli

@@ -59,14 +59,14 @@ fi
 if [ "$COMMAND" = "ckan" ]; then
     # adjust args to match ckan expectations
     COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/')
-    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2
     exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS
 elif [ "$COMMAND" = "paster" ]; then
     # adjust args to match paster expectations
     COMMAND=$1
-    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
     shift
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND $1..." >&2
     if [ "$1" = "show" ]; then shift; fi
     exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI}
 else

bin/create-test-data.sh

@@ -2,7 +2,7 @@
 ##
 # Create some example content for extension BDD tests.
 #
-set -e
+set -ex
 
 CKAN_ACTION_URL=${CKAN_SITE_URL}api/action
 CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
@@ -43,27 +43,25 @@ add_user_if_needed test_org_member "Test Member" test_org_member@localhost
 
 echo "Creating ${TEST_ORG_TITLE} organisation:"
 
+api_call () {
+    wget -O - --header="Authorization: ${API_KEY}" --post-data "$1" ${CKAN_ACTION_URL}/$2
+}
+
 TEST_ORG=$( \
-    curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'"}' \
-    ${CKAN_ACTION_URL}/organization_create
+    api_call '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'",
+        "description": "Organisation for testing issues"}' organization_create
 )
 
 TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py)
 
 echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):"
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \
-    ${CKAN_ACTION_URL}/member_create
+api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' member_create
+
+api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' member_create
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \
-    ${CKAN_ACTION_URL}/member_create
+api_call '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' member_create
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \
-    ${CKAN_ACTION_URL}/member_create
 ##
 # END.
 #
@@ -83,31 +81,23 @@ add_user_if_needed report_editor "Reporting Editor" report_editor@localhost
 echo "Creating ${REPORT_ORG_TITLE} Organisation:"
 
 REPORT_ORG=$( \
-    curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' \
-    ${CKAN_ACTION_URL}/organization_create
+    api_call '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' organization_create
 )
 
 REPORT_ORG_ID=$(echo $REPORT_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py)
 
 echo "Assigning test users to ${REPORT_ORG_TITLE} Organisation:"
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' \
-    ${CKAN_ACTION_URL}/member_create
+api_call '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' member_create
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_editor", "object_type": "user", "capacity": "editor"}' \
-    ${CKAN_ACTION_URL}/member_create
+api_call '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_editor", "object_type": "user", "capacity": "editor"}' member_create
 
 echo "Creating test dataset for reporting:"
 
-curl -LsH "Authorization: ${API_KEY}" \
-    --data '{"name": "reporting", "title": "Reporting dataset", "description": "Dataset for testing reports",
+api_call '{"name": "reporting", "title": "Reporting dataset", "description": "Dataset for testing reports",
 "owner_org": "'"${REPORT_ORG_ID}"'", "update_frequency": "near-realtime", "author_email": "report_admin@localhost",
 "version": "1.0", "license_id": "cc-by-4", "data_driven_application": "NO", "security_classification": "PUBLIC",
-"notes": "test", "de_identified_data": "NO"}'\
-    ${CKAN_ACTION_URL}/package_create
+"notes": "test", "de_identified_data": "NO"}' package_create
 
 ##
 # END.