Merge pull request ckan#48 from qld-gov-au/QOLDEV-323-ckan-2.10

QOLDEV-323 prepare for CKAN 2.10

.ahoy.yml

@@ -70,17 +70,17 @@ commands:
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/scripts/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/scripts/activate && cd $APP_DIR && sh'; fi
+    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi
 
   doctor:
     usage: Find problems with current project setup.
-    cmd: .docker/scripts/doctor.sh "$@"
+    cmd: bin/doctor.sh "$@"
 
   install-site:
-    usage: Install a site.
+    usage: Install test site data.
     cmd: |
       ahoy title "Installing a fresh site"
-      ahoy cli '$APP_DIR/scripts/init.sh'
+      ahoy cli '$APP_DIR/bin/init.sh'
 
   clean:
     usage: Remove containers and all build files.
@@ -113,8 +113,7 @@ commands:
     usage: Update files from local repo.
     cmd: |
       docker cp . $(docker-compose ps -q ckan):/srv/app/
-      docker cp .docker/scripts $(docker-compose ps -q ckan):/srv/app/
-      docker cp .docker/scripts/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
+      docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
       ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test.ini $CKAN_INI'
 
   test-unit:

.circleci/config.yml

@@ -24,13 +24,13 @@ jobs:
       - checkout
       - *step_setup_remote_docker
       - run:
-          command: .circleci/build.sh
+          command: bin/build.sh
           environment:
             CKAN_VERSION: 2.9
-      - run: .circleci/test.sh
+      - run: bin/test.sh
       - run:
           name: Process artifacts
-          command: .circleci/process-artifacts.sh
+          command: bin/process-artifacts.sh
           when: always
       - store_artifacts:
           path: /tmp/artifacts

.docker/Dockerfile-template.ckan

@@ -1,33 +1,29 @@
-FROM openknowledge/ckan-dev:@CKAN_VERSION@
+FROM openknowledge/ckan-dev:{CKAN_VERSION}
 
 ARG SITE_URL=http://ckan:5000/
-ENV PYTHON_VERSION=@PYTHON_VERSION@
-ENV CKAN_VERSION=@CKAN_VERSION@
+ENV CKAN_VERSION={CKAN_VERSION}
+ENV PYTHON_VERSION={PYTHON_VERSION}
 ENV CKAN_SITE_URL="${SITE_URL}"
+ENV PYTHON={PYTHON}
 
 WORKDIR "${APP_DIR}"
 
 ENV DOCKERIZE_VERSION v0.6.1
 RUN apk add --no-cache build-base \
-    && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
-    && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
-    && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
-
-# Install CKAN.
+    && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+        | tar -C /usr/local/bin -xzvf -
 
 COPY .docker/test.ini $CKAN_INI
 
 RUN sed -i "s@SITE_URL@${CKAN_SITE_URL}@g" $CKAN_INI
 
 COPY . ${APP_DIR}/
 
-COPY .docker/scripts ${APP_DIR}/scripts
-
-COPY .docker/scripts/ckan_cli /usr/bin/
+COPY bin/ckan_cli /usr/bin/
 
-RUN chmod +x ${APP_DIR}/scripts/*.sh /usr/bin/ckan_cli
+RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
 
 # Init current extension.
-RUN ${APP_DIR}/scripts/init-ext.sh
+RUN ${APP_DIR}/bin/init-ext.sh
 
-CMD ["/srv/app/scripts/serve.sh"]
+CMD ["/srv/app/bin/serve.sh"]

.docker/test.ini

@@ -13,7 +13,7 @@
 
 [DEFAULT]
 debug = false
-smtp_server = localhost
+smtp_server = localhost:8025
 error_email_from = paste@localhost
 
 [server:main]
@@ -55,6 +55,7 @@ ckan.datastore.default_fts_index_method = gist
 
 ## Site Settings.
 ckan.site_url = http://ckan:5000/
+WTF_CSRF_ENABLED = False
 
 ckan.auth.create_unowned_dataset=true
 

.flake8

@@ -3,7 +3,6 @@
 
 exclude =
     ckan
-    scripts
 
 # Extended output format.
 format = pylint

.github/workflows/ci.yml → .github/workflows/test.yml

@@ -5,15 +5,14 @@ on:
   pull_request:
     branches:
       - master
-      - develop
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         timeout-minutes: 2
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         timeout-minutes: 5
         with:
           python-version: '3.x'
@@ -29,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ckan-version: [2.9, "2.9-py2"]
+        ckan-version: ["2.10", 2.9, 2.9-py2]
 
     name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
@@ -38,25 +37,25 @@ jobs:
       CKAN_VERSION: ${{ matrix.ckan-version }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         timeout-minutes: 2
 
       - name: Build
-        run: .circleci/build.sh
+        run: bin/build.sh
         timeout-minutes: 15
 
       - name: Test
-        run: .circleci/test.sh
+        run: bin/test.sh
         timeout-minutes: 20
 
       - name: Retrieve screenshots
         if: failure()
-        run: .circleci/process-artifacts.sh
+        run: bin/process-artifacts.sh
         timeout-minutes: 1
 
       - name: Upload screenshots
         if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots

.circleci/build.sh → bin/build.sh

@@ -2,7 +2,7 @@
 ##
 # Build site in CI.
 #
-set -e
+set -ex
 
 # Process Docker Compose configuration. This is used to avoid multiple
 # docker-compose.yml files.
@@ -21,13 +21,17 @@ export DOCTOR_CHECK_SSH=0
 export DOCTOR_CHECK_WEBSERVER=0
 export DOCTOR_CHECK_BOOTSTRAP=0
 
+PYTHON=python
 if [ "$CKAN_VERSION" = "2.9-py2" ]; then
     PYTHON_VERSION=py2
 else
     PYTHON_VERSION=py3
+    PYTHON="${PYTHON}3"
 fi
 
-sed "s|@CKAN_VERSION@|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
-    | sed "s|@PYTHON_VERSION@|$PYTHON_VERSION|g" > .docker/Dockerfile.ckan
+sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
+    | sed "s|{PYTHON}|$PYTHON|g" \
+    | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" \
+    > .docker/Dockerfile.ckan
 
 ahoy build || (ahoy logs; exit 1)

.docker/scripts/create-test-data.sh → bin/create-test-data.sh

@@ -4,17 +4,12 @@
 #
 set -e
 
-if [ "$PYTHON_VERSION" = "py3" ]; then
-    PYTHON=python3
-else
-    PYTHON=python
-fi
+CKAN_ACTION_URL=${CKAN_SITE_URL}api/action
 CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
 CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
 CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
-CKAN_ACTION_URL=${CKAN_SITE_URL}api/action
 
-. ${APP_DIR}/scripts/activate
+. ${APP_DIR}/bin/activate
 
 add_user_if_needed () {
     echo "Adding user '$2' ($1) with email address [$3]"
@@ -30,7 +25,7 @@ ckan_cli sysadmin add "${CKAN_USER_NAME}"
 API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
 if [ "$API_KEY" = "None" ]; then
     echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..."
-    API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |grep -v '^API Token created' | tr -d '[:space:]')
+    API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |tail -1 | tr -d '[:space:]')
 fi
 
 ##
@@ -50,24 +45,24 @@ echo "Creating ${TEST_ORG_TITLE} organisation:"
 
 TEST_ORG=$( \
     curl -LsH "Authorization: ${API_KEY}" \
-    --data "name=${TEST_ORG_NAME}&title=${TEST_ORG_TITLE}" \
+    --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'"}' \
     ${CKAN_ACTION_URL}/organization_create
 )
 
-TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON $APP_DIR/scripts/extract-id.py)
+TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py)
 
 echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):"
 
 curl -LsH "Authorization: ${API_KEY}" \
-    --data "id=${TEST_ORG_ID}&object=test_org_admin&object_type=user&capacity=admin" \
+    --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \
     ${CKAN_ACTION_URL}/member_create
 
 curl -LsH "Authorization: ${API_KEY}" \
-    --data "id=${TEST_ORG_ID}&object=test_org_editor&object_type=user&capacity=editor" \
+    --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \
     ${CKAN_ACTION_URL}/member_create
 
 curl -LsH "Authorization: ${API_KEY}" \
-    --data "id=${TEST_ORG_ID}&object=test_org_member&object_type=user&capacity=member" \
+    --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \
     ${CKAN_ACTION_URL}/member_create
 ##
 # END.
@@ -79,4 +74,4 @@ curl -LsH "Authorization: ${API_KEY}" \
 "author_email": "admin@localhost", "license_id": "other-open", "notes": "test"}' \
     ${CKAN_ACTION_URL}/package_create
 
-. ${APP_DIR}/scripts/deactivate
+. ${APP_DIR}/bin/deactivate

.docker/scripts/init-ext.sh → bin/init-ext.sh

@@ -2,7 +2,7 @@
 ##
 # Install current extension.
 #
-set -e
+set -ex
 
 install_requirements () {
     PROJECT_DIR=$1
@@ -26,7 +26,7 @@ install_requirements () {
     done
 }
 
-. ${APP_DIR}/scripts/activate
+. ${APP_DIR}/bin/activate
 
 
 if [ "$CKAN_VERSION" = "2.9-py2" ]; then
@@ -45,4 +45,4 @@ installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1
 # Validate that the extension was installed correctly.
 if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi
 
-. ${APP_DIR}/scripts/deactivate
+. ${APP_DIR}/bin/deactivate

.docker/scripts/init.sh → bin/init.sh

@@ -4,7 +4,7 @@
 #
 set -e
 
-. ${APP_DIR}/scripts/activate
+. ${APP_DIR}/bin/activate
 CLICK_ARGS="--yes" ckan_cli db clean
 ckan_cli db init
 ckan_cli db upgrade
@@ -13,4 +13,4 @@ ckan_cli db upgrade
 PASTER_PLUGIN=ckanext-validation ckan_cli validation init-db
 
 # Create some base test data
-. $APP_DIR/scripts/create-test-data.sh
+. $APP_DIR/bin/create-test-data.sh

.docker/scripts/serve.sh → bin/serve.sh

@@ -5,7 +5,7 @@ dockerize -wait tcp://postgres:5432 -timeout 1m
 dockerize -wait tcp://solr:8983 -timeout 1m
 dockerize -wait tcp://redis:6379 -timeout 1m
 
-. ${APP_DIR}/scripts/activate
+. ${APP_DIR}/bin/activate
 if (which ckan > /dev/null); then
     ckan -c ${CKAN_INI} run -r
 else

.circleci/test.sh → bin/test.sh

@@ -12,4 +12,5 @@ ahoy test-unit
 
 echo "==> Run BDD tests"
 ahoy install-site
+ahoy cli "rm -r test/screenshots || true"
 ahoy test-bdd || (ahoy logs; exit 1)

ckanext/validation/helpers.py

@@ -22,7 +22,9 @@ def _get_helpers():
 def get_validation_badge(resource, in_listing=False):
 
     afterDate = config.get('ckanext.validation.show_badges_after_last_modified_date', "")
-    if afterDate and h.date_str_to_datetime(afterDate) >= h.date_str_to_datetime(resource['last_modified']):
+    if afterDate and (not resource.get('last_modified')
+                      or h.date_str_to_datetime(afterDate)
+                      >= h.date_str_to_datetime(resource['last_modified'])):
         return ''
 
     if in_listing and not asbool(

ckanext/validation/templates/package/resource_read.html

@@ -6,8 +6,7 @@ <h1 class="page-heading">{{ h.resource_display_name(res) | truncate(50) }}
   {{ h.get_validation_badge(res)|safe }}
   </h1>
 
-  {% set type = 'asset' %}
-  {% include 'validation/snippets/validation_style_' ~ type ~ '.html' %}
+  {% asset 'ckanext-validation/main-css' %}
 
 {% endblock %}
 

ckanext/validation/templates/package/snippets/resource_item.html

@@ -4,8 +4,6 @@
 {{ super() }}
 {{ h.get_validation_badge(res, in_listing=True)|safe }}
 
-{% set type = 'asset' %}
-{% include 'validation/snippets/validation_style_' ~ type ~ '.html' %}
-{% endblock %}
-
+  {% asset 'ckanext-validation/main-css' %}
 
+{% endblock %}

ckanext/validation/templates/scheming/form_snippets/resource_schema.html

@@ -64,7 +64,6 @@
     {% set existing_value = h.scheming_display_json_value(value, indent=None) if is_json else value %}
     <input type="hidden" id="field-schema" name="schema" value="{{ existing_value }}" />
 
-    {% set type = 'asset' %}
-    {% include 'validation/snippets/validation_resource-schema-form_' ~ type ~ '.html' %}
+    {% asset 'ckanext-validation/resource-schema-form' %}
 
   </div>