Merge pull request #138 from qld-gov-au/develop

Develop to master - fix email template and update tests

.ahoy.yml

@@ -28,51 +28,59 @@ commands:
     cmd: |
       ahoy line "Project                  : " ${PROJECT}
       ahoy line "Site local URL           : " ${LAGOON_LOCALDEV_URL}
-      ahoy line "DB port on host          : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
-      ahoy line "Solr port on host        : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+      ahoy line "DB port on host          : " $(docker port $(sh bin/docker-compose.sh ps -q postgres) 5432 | cut -d : -f 2)
+      ahoy line "Solr port on host        : " $(docker port $(sh bin/docker-compose.sh ps -q solr) 8983 | cut -d : -f 2)
       ahoy line "Mailhog URL              : " http://mailhog.docker.amazee.io/
 
   up:
     usage: Build and start Docker containers.
     cmd: |
       ahoy title "Building and starting Docker containers"
-      docker-compose up -d "$@"
+      sh bin/docker-compose.sh up -d "$@"
       echo "Initialising database schema"
       ahoy cli '$APP_DIR/bin/init.sh'
       echo "Waiting for containers to start listening..."
       ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
-      if docker-compose logs | grep -q "\[Error\]"; then exit 1; fi
-      if docker-compose logs | grep -q "Exception"; then exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "Exception"; then exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
       export DOCTOR_CHECK_CLI=0
 
   down:
     usage: Stop Docker containers and remove container, images, volumes and networks.
-    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
+    cmd: |
+      ahoy title 'Stopping and removing old containers, images, volumes, networks'
+      if [ -f "docker-compose.yml" ]; then sh bin/docker-compose.sh down --volumes; fi
 
   start:
     usage: Start existing Docker containers.
-    cmd: docker-compose start "$@"
+    cmd: sh bin/docker-compose.sh start "$@"
 
   stop:
     usage: Stop running Docker containers.
-    cmd: docker-compose stop "$@"
+    cmd: sh bin/docker-compose.sh stop "$@"
 
   restart:
     usage: Restart all stopped and running Docker containers.
-    cmd: docker-compose restart "$@"
+    cmd: sh bin/docker-compose.sh restart "$@"
 
   logs:
     usage: Show Docker logs.
-    cmd: docker-compose logs "$@"
+    cmd: sh bin/docker-compose.sh logs "$@"
 
   pull:
     usage: Pull latest docker images.
     cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep ckan/ckan- | grep -v none | xargs -n1 docker pull | cat; fi
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi
+    cmd: |
+      CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
+      if [ "${#}" -ne 0 \]; then
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
+      else
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
+      fi
 
   doctor:
     usage: Find problems with current project setup.
@@ -104,7 +112,7 @@ commands:
 
   flush-redis:
     usage: Flush Redis cache.
-    cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+    cmd: docker exec -i $(sh bin/docker-compose.sh ps -q redis) redis-cli flushall > /dev/null
 
   lint:
     usage: Lint code.
@@ -116,8 +124,8 @@ commands:
   copy-local-files:
     usage: Update files from local repo.
     cmd: |
-      docker cp . $(docker-compose ps -q ckan):/srv/app/
-      docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
+      docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
+      docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
       ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI; $APP_DIR/bin/process-config.sh'
 
   pip-list:

.docker/test.ini

@@ -29,6 +29,7 @@ use = egg:ckan
 full_stack = true
 cache_dir = /tmp/%(ckan.site_id)s/
 beaker.session.key = ckan
+beaker.session.type = file
 
 # This is the secret token that the beaker library uses to hash the cookie sent
 # to the client. `paster make-config` generates a unique value for this each
@@ -54,27 +55,11 @@ ckan.datastore.default_fts_index_method = gist
 ## Site Settings.
 ckan.site_url = http://ckan:5000/
 
-## Authorization Settings
-
-ckan.auth.anon_create_dataset = false
-ckan.auth.create_unowned_dataset = false
-ckan.auth.create_dataset_if_not_in_organization = false
-ckan.auth.user_create_groups = false
-ckan.auth.user_create_organizations = false
-ckan.auth.user_delete_groups = true
-ckan.auth.user_delete_organizations = true
-ckan.auth.create_user_via_api = false
-ckan.auth.create_user_via_web = true
-ckan.auth.roles_that_cascade_to_sub_groups = admin
-ckan.auth.public_user_details = True
-
-
 ## Search Settings
 
 ckan.site_id = default
 solr_url = http://solr:8983/solr/ckan
 
-
 ## Redis Settings
 
 # URL to your Redis instance, including the database to be used.
@@ -88,15 +73,27 @@ ckan.redis.url = redis://redis:6379
 # ckan.cors.origin_allow_all = true
 # cors.origin_whitelist is a space separated list of allowed domains.
 # ckan.cors.origin_whitelist = http://example1.com http://example2.com
+## Authorization Settings
 
+ckan.auth.anon_create_dataset = false
+ckan.auth.create_unowned_dataset = false
+ckan.auth.create_dataset_if_not_in_organization = false
+ckan.auth.user_create_groups = false
+ckan.auth.user_create_organizations = false
+ckan.auth.user_delete_groups = true
+ckan.auth.user_delete_organizations = true
+ckan.auth.create_user_via_api = false
+ckan.auth.create_user_via_web = true
+ckan.auth.roles_that_cascade_to_sub_groups = admin
+ckan.auth.public_user_details = True
 
 ## Plugins Settings
 
 # Note: Add ``datastore`` to enable the CKAN DataStore
 #       Add ``datapusher`` to enable DataPusher
 #       Add ``resource_proxy`` to enable resource proxying and get around the
 #       same origin policy
-ckan.plugins = stats text_view image_view recline_view qgovext
+ckan.plugins = qgovext
 
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)

.gitattributes

@@ -1,6 +1,5 @@
 # Ignore files for distribution archives.
 /.ahoy.yml           export-ignore
-/.circleci           export-ignore
 /.docker             export-ignore
 /.env                export-ignore
 /.gitatributes       export-ignore

.github/workflows/test.yml

@@ -30,7 +30,7 @@ jobs:
 
     name: CKAN ${{ matrix.ckan-version }} ${{ matrix.ckan-type }}
     runs-on: ubuntu-latest
-    container: drevops/ci-builder
+    container: drevops/ci-builder:23.7.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
       CKAN_TYPE: ${{ matrix.ckan-type }}

bin/build.sh

@@ -21,25 +21,25 @@ CKAN_GIT_ORG=ckan
 
 if [ "$CKAN_VERSION" = "2.10" ]; then
     if [ "$CKAN_TYPE" = "custom" ]; then
-      CKAN_GIT_VERSION=ckan-2.10.0-qgov.1
-      CKAN_GIT_ORG=qld-gov-au
+        CKAN_GIT_VERSION=ckan-2.10.3-qgov.3
+        CKAN_GIT_ORG=qld-gov-au
     fi
 
     PYTHON_VERSION=py3
     PYTHON="${PYTHON}3"
 else
-    if [ "$CKAN_TYPE" = "custom" ]; then
-      CKAN_GIT_VERSION=ckan-2.9.5-qgov.9
-      CKAN_GIT_ORG=qld-gov-au
-    fi
-
     if [ "$CKAN_VERSION" = "2.9-py2" ]; then
         PYTHON_VERSION=py2
-        CKAN_GIT_VERSION=2.9
     else
         PYTHON_VERSION=py3
         PYTHON="${PYTHON}3"
     fi
+    if [ "$CKAN_TYPE" = "custom" ]; then
+        CKAN_GIT_VERSION=ckan-2.9.9-qgov.4
+        CKAN_GIT_ORG=qld-gov-au
+    else
+        CKAN_GIT_VERSION=ckan-2.9.9
+    fi
 fi
 
 sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \

bin/docker-compose.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# Pass commands to Docker Compose v1 or v2 depending on what is present
+
+if (which docker-compose >/dev/null); then
+    # Docker Compose v1
+    docker-compose $*
+elif (docker compose ls >/dev/null); then
+    # Docker Compose v2
+    docker compose $*
+else
+    # Docker Compose not found
+    exit 1
+fi

bin/doctor.sh

@@ -31,7 +31,6 @@ main() {
 
   if [ "${DOCTOR_CHECK_TOOLS}" == "1" ]; then
     [ "$(command_exists docker)" == "1" ] && error "Please install Docker (https://www.docker.com/get-started)" && exit 1
-    [ "$(command_exists docker-compose)" == "1" ] && error "Please install docker-compose (https://docs.docker.com/compose/install/)" && exit 1
     [ "$(command_exists composer)" == "1" ] && error "Please install Composer (https://getcomposer.org/)" && exit 1
     [ "$(command_exists pygmy)" == "1" ] && error "Please install Pygmy (https://pygmy.readthedocs.io/)" && exit 1
     [ "$(command_exists ahoy)" == "1" ] && error "Please install Ahoy (https://ahoy-cli.readthedocs.io/)" && exit 1
@@ -55,7 +54,7 @@ main() {
 
   # Check that the stack is running.
   if [ "${DOCTOR_CHECK_CLI}" == "1" ]; then
-    if ! docker ps -q --no-trunc | grep "$(docker-compose ps -q ckan)" > /dev/null 2>&1; then
+    if ! docker ps -q --no-trunc | grep "$(sh bin/docker-compose.sh ps -q ckan)" > /dev/null 2>&1; then
       error "CLI container is not running. Run 'ahoy up'."
       exit 1
     fi
@@ -94,7 +93,7 @@ main() {
     fi
 
     # Check that the volume is mounted into CLI container.
-    if ! docker exec -i "$(docker-compose ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then
+    if ! docker exec -i "$(sh bin/docker-compose.sh ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then
       error "SSH key is added to Pygmy, but the volume is not mounted into container. Make sure that your your \"docker-compose.yml\" has the following lines:"
       error "volumes_from:"
       error "  - container:amazeeio-ssh-agent"
@@ -103,7 +102,7 @@ main() {
     fi
 
     # Check that ssh key is available in the container.
-    if ! docker exec -i "$(docker-compose ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then
+    if ! docker exec -i "$(sh bin/docker-compose.sh ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then
       error "SSH key was not added into container. Run 'ahoy up -- --build'."
       exit 1
     fi
@@ -113,7 +112,7 @@ main() {
 
 
   if [ "${DOCTOR_CHECK_WEBSERVER}" == "1" ]; then
-    host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)"
+    host_app_port="$(docker port $(sh bin/docker-compose.sh ps -q ckan) $APP_PORT | cut -d : -f 2)"
     if ! curl -L -s -o /dev/null -w "%{http_code}" "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q 200; then
       error "Web server is not accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}"
       exit 1
@@ -122,7 +121,7 @@ main() {
   fi
 
   if [ "${DOCTOR_CHECK_BOOTSTRAP}" == "1" ]; then
-    host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)"
+    host_app_port="$(docker port $(sh bin/docker-compose.sh ps -q ckan) $APP_PORT | cut -d : -f 2)"
     if ! curl -L -s -N "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q -i "meta name=\"generator\" content=\"ckan"; then
       error "Website is running, but cannot be bootstrapped. Try pulling latest container images with 'ahoy pull'"
       exit 1

bin/process-artifacts.sh

@@ -10,4 +10,4 @@ ahoy cli "mkdir -p test/screenshots"
 
 # Copy from the app container to the build host for storage.
 mkdir -p /tmp/artifacts/behave
-docker cp "$(docker-compose ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/
+docker cp "$(sh bin/docker-compose.sh ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/

ckanext/qgov/common/resources/urlm.json

@@ -35,13 +35,9 @@
     "url" : "https://test.smartservice.qld.gov.au/services/url/translate/v3.json?sourceurl={source}",
     "proxy" : null
   },
-  "data-local" : {
+  "ckan:5000" : {
     "url" : "https://test.smartservice.qld.gov.au/services/url/translate/v3.json?sourceurl={source}",
-    "proxy" : "proxy:3128"
-  },
-  "epub-local" : {
-    "url" : "https://test.smartservice.qld.gov.au/services/url/translate/v3.json?sourceurl={source}",
-    "proxy" : "proxy:3128"
+    "proxy" : null
   },
   "127.0.0.1:5000" : {
     "url" : "https://test.smartservice.qld.gov.au/services/url/translate/v3.json?sourceurl={source}",

ckanext/qgov/common/templates/activity_streams/activity_stream_email_notifications.text

@@ -1,25 +1,23 @@
-{%- set num = activities|length -%}
+{% set num = activities|length %}{{ ngettext("You have {num} new activity on your {site_title} dashboard", "You have {num} new activities on your {site_title} dashboard", num).format(site_title=g.site_title if g else site_title, num=num) }} {{ _('To view your dashboard, click on this link:') }}
 
-{{ ungettext("You have {num} new activity on your {site_title} dashboard", "You have {num} new activities on your {site_title} dashboard", num).format(site_title=g.site_title, num=num) }}:
+{% url_for 'activity.dashboard', _external=True %}
 
 {% for activity in activities -%}
     {%- set data = activity['data'] if activity['data'] else None -%}
     {%- set activity_type = activity['activity_type'] if activity['activity_type'] else None -%}
+    {%- set id = activity['object_id'] -%}
     {%- if data -%}
         {%- if data['package'] -%}
-            {%- set id = data['package']['name'] -%}
-            {%- set action = 'dataset_read' -%}
+            {%- set name = data['package']['title'] -%}
+            {%- set action = 'dataset.read' -%}
         {%- elif data['group'] -%}
-            {%- set id = data['group']['name'] -%}
-            {%- set action = 'organization_read' if data['group']['is_organization'] else 'group_read' -%}
+            {%- set name = data['group']['title'] -%}
+            {%- set action = 'organization.read' if activity_type == 'changed organization' else 'group.read' -%}
         {%- endif -%}
     {%- endif -%}
-    {% if action and id %}{{ h.url_for(action, id=id, _external=True) }}{% if activity_type %} ({{ h.activity_type_nice(activity_type)|capitalize }}){% endif %}{% endif %}
+    {% if action and id %}{{name}} {% if activity_type %}({{ (activity_type|replace('organization', _('organization'))|replace('package', 'dataset')|capitalize).split()|reverse|join(' ') }}){% endif %} {% url_for action, id=id, _external=True %}{% endif %}
 {% endfor %}
-{{ _('To view your dashboard, click on this link:') }}
 
-{{ g.site_url + '/dashboard' }}
+{{ _('You can turn off these email notifications in your {site_title} preferences. To change your preferences, click on this link:').format(site_title=g.site_title if g else site_title) }}
 
-{{ _('You can turn off these email notifications in your {site_title} preferences. To change your preferences, click on this link:').format(site_title=g.site_title) }}
-
-{{ g.site_url + '/user/edit' }}
+{% url_for 'user.edit', _external=True %}

ckanext/qgov/common/test/test_authenticator.py

@@ -1,8 +1,9 @@
 # encoding: utf-8
 
 import pytest
+
 from ckan.model import User
-import ckan.lib.create_test_data as ctd
+from ckan.lib import authenticator as core_authenticator, create_test_data as ctd
 from ckan.plugins.toolkit import check_ckan_version
 
 if check_ckan_version('2.10'):
@@ -17,6 +18,15 @@ def qgov_authenticate(identity):
 CreateTestData = ctd.CreateTestData
 
 
+class MockGlobal(object):
+
+    def __init__(self):
+        self.recaptcha_publickey = None
+
+
+core_authenticator.g = MockGlobal()
+
+
 @pytest.mark.usefixtures("clean_db")
 class TestUsernamePasswordAuthenticator(object):
 

test/features/environment.py

@@ -103,4 +103,5 @@ def before_scenario(context, scenario):
 
 
 def after_scenario(context, scenario):
+    os.system("ckan_cli jobs clear")
     benv.after_scenario(context, scenario)