diff --git a/.ahoy.yml b/.ahoy.yml index 3ef6fdc2..a9af6d93 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -70,17 +70,17 @@ commands: cli: usage: Start a shell inside CLI container or run a command. - cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/scripts/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/scripts/activate && cd $APP_DIR && sh'; fi + cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi doctor: usage: Find problems with current project setup. - cmd: .docker/scripts/doctor.sh "$@" + cmd: bin/doctor.sh "$@" install-site: - usage: Install a site. + usage: Install test site data. cmd: | ahoy title "Installing a fresh site" - ahoy cli '$APP_DIR/scripts/init.sh' + ahoy cli '$APP_DIR/bin/init.sh' clean: usage: Remove containers and all build files. @@ -113,8 +113,7 @@ commands: usage: Update files from local repo. cmd: | docker cp . $(docker-compose ps -q ckan):/srv/app/ - docker cp .docker/scripts $(docker-compose ps -q ckan):/srv/app/ - docker cp .docker/scripts/ckan_cli $(docker-compose ps -q ckan):/usr/bin/ + docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/ ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test.ini $CKAN_INI' test-unit: diff --git a/.circleci/config.yml b/.circleci/config.yml index c51ccacb..018731b1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,13 +24,13 @@ jobs: - checkout - *step_setup_remote_docker - run: - command: .circleci/build.sh + command: bin/build.sh environment: CKAN_VERSION: 2.9 - - run: .circleci/test.sh + - run: bin/test.sh - run: name: Process artifacts - command: .circleci/process-artifacts.sh + command: bin/process-artifacts.sh when: always - store_artifacts: path: /tmp/artifacts diff --git a/.docker/Dockerfile-template.ckan b/.docker/Dockerfile-template.ckan index b9cc7130..523140fa 100644 --- a/.docker/Dockerfile-template.ckan +++ b/.docker/Dockerfile-template.ckan @@ -1,19 +1,17 @@ -FROM openknowledge/ckan-dev:@CKAN_VERSION@ +FROM openknowledge/ckan-dev:{CKAN_VERSION} ARG SITE_URL=http://ckan:5000/ -ENV PYTHON_VERSION=@PYTHON_VERSION@ -ENV CKAN_VERSION=@CKAN_VERSION@ +ENV CKAN_VERSION={CKAN_VERSION} +ENV PYTHON_VERSION={PYTHON_VERSION} ENV CKAN_SITE_URL="${SITE_URL}" +ENV PYTHON={PYTHON} WORKDIR "${APP_DIR}" ENV DOCKERIZE_VERSION v0.6.1 RUN apk add --no-cache build-base \ - && curl -sLO https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ - && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ - && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz - -# Install CKAN. + && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ + | tar -C /usr/local/bin -xzvf - COPY .docker/test.ini $CKAN_INI @@ -21,13 +19,11 @@ RUN sed -i "s@SITE_URL@${CKAN_SITE_URL}@g" $CKAN_INI COPY . ${APP_DIR}/ -COPY .docker/scripts ${APP_DIR}/scripts - -COPY .docker/scripts/ckan_cli /usr/bin/ +COPY bin/ckan_cli /usr/bin/ -RUN chmod +x ${APP_DIR}/scripts/*.sh /usr/bin/ckan_cli +RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli # Init current extension. -RUN ${APP_DIR}/scripts/init-ext.sh +RUN ${APP_DIR}/bin/init-ext.sh -CMD ["/srv/app/scripts/serve.sh"] +CMD ["/srv/app/bin/serve.sh"] diff --git a/.docker/test.ini b/.docker/test.ini index ad0f79b2..88911e88 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -13,7 +13,7 @@ [DEFAULT] debug = false -smtp_server = localhost +smtp_server = localhost:8025 error_email_from = paste@localhost [server:main] @@ -55,6 +55,7 @@ ckan.datastore.default_fts_index_method = gist ## Site Settings. ckan.site_url = http://ckan:5000/ +WTF_CSRF_ENABLED = False ckan.auth.create_unowned_dataset=true diff --git a/.flake8 b/.flake8 index e81d4e69..55f2a01d 100644 --- a/.flake8 +++ b/.flake8 @@ -3,7 +3,6 @@ exclude = ckan - scripts # Extended output format. format = pylint diff --git a/.github/workflows/ci.yml b/.github/workflows/test.yml similarity index 78% rename from .github/workflows/ci.yml rename to .github/workflows/test.yml index 87ad4c53..b4f16005 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/test.yml @@ -5,15 +5,14 @@ on: pull_request: branches: - master - - develop jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 timeout-minutes: 2 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 timeout-minutes: 5 with: python-version: '3.x' @@ -29,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - ckan-version: [2.9, "2.9-py2"] + ckan-version: ["2.10", 2.9, 2.9-py2] name: Continuous Integration build on CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest @@ -38,25 +37,25 @@ jobs: CKAN_VERSION: ${{ matrix.ckan-version }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 timeout-minutes: 2 - name: Build - run: .circleci/build.sh + run: bin/build.sh timeout-minutes: 15 - name: Test - run: .circleci/test.sh + run: bin/test.sh timeout-minutes: 20 - name: Retrieve screenshots if: failure() - run: .circleci/process-artifacts.sh + run: bin/process-artifacts.sh timeout-minutes: 1 - name: Upload screenshots if: failure() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: CKAN ${{ matrix.ckan-version }} screenshots path: /tmp/artifacts/behave/screenshots diff --git a/.docker/scripts/activate b/bin/activate similarity index 100% rename from .docker/scripts/activate rename to bin/activate diff --git a/.circleci/build.sh b/bin/build.sh similarity index 74% rename from .circleci/build.sh rename to bin/build.sh index c2bb9abc..8ffe840a 100755 --- a/.circleci/build.sh +++ b/bin/build.sh @@ -2,7 +2,7 @@ ## # Build site in CI. # -set -e +set -ex # Process Docker Compose configuration. This is used to avoid multiple # docker-compose.yml files. @@ -21,13 +21,17 @@ export DOCTOR_CHECK_SSH=0 export DOCTOR_CHECK_WEBSERVER=0 export DOCTOR_CHECK_BOOTSTRAP=0 +PYTHON=python if [ "$CKAN_VERSION" = "2.9-py2" ]; then PYTHON_VERSION=py2 else PYTHON_VERSION=py3 + PYTHON="${PYTHON}3" fi -sed "s|@CKAN_VERSION@|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \ - | sed "s|@PYTHON_VERSION@|$PYTHON_VERSION|g" > .docker/Dockerfile.ckan +sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \ + | sed "s|{PYTHON}|$PYTHON|g" \ + | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" \ + > .docker/Dockerfile.ckan ahoy build || (ahoy logs; exit 1) diff --git a/.docker/scripts/ckan_cli b/bin/ckan_cli similarity index 100% rename from .docker/scripts/ckan_cli rename to bin/ckan_cli diff --git a/.docker/scripts/create-test-data.sh b/bin/create-test-data.sh similarity index 78% rename from .docker/scripts/create-test-data.sh rename to bin/create-test-data.sh index 1eb43d50..64ebd324 100644 --- a/.docker/scripts/create-test-data.sh +++ b/bin/create-test-data.sh @@ -4,17 +4,12 @@ # set -e -if [ "$PYTHON_VERSION" = "py3" ]; then - PYTHON=python3 -else - PYTHON=python -fi +CKAN_ACTION_URL=${CKAN_SITE_URL}api/action CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" -CKAN_ACTION_URL=${CKAN_SITE_URL}api/action -. ${APP_DIR}/scripts/activate +. ${APP_DIR}/bin/activate add_user_if_needed () { echo "Adding user '$2' ($1) with email address [$3]" @@ -30,7 +25,7 @@ ckan_cli sysadmin add "${CKAN_USER_NAME}" API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') if [ "$API_KEY" = "None" ]; then echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..." - API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |grep -v '^API Token created' | tr -d '[:space:]') + API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |tail -1 | tr -d '[:space:]') fi ## @@ -50,24 +45,24 @@ echo "Creating ${TEST_ORG_TITLE} organisation:" TEST_ORG=$( \ curl -LsH "Authorization: ${API_KEY}" \ - --data "name=${TEST_ORG_NAME}&title=${TEST_ORG_TITLE}" \ + --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'"}' \ ${CKAN_ACTION_URL}/organization_create ) -TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON $APP_DIR/scripts/extract-id.py) +TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py) echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):" curl -LsH "Authorization: ${API_KEY}" \ - --data "id=${TEST_ORG_ID}&object=test_org_admin&object_type=user&capacity=admin" \ + --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \ ${CKAN_ACTION_URL}/member_create curl -LsH "Authorization: ${API_KEY}" \ - --data "id=${TEST_ORG_ID}&object=test_org_editor&object_type=user&capacity=editor" \ + --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \ ${CKAN_ACTION_URL}/member_create curl -LsH "Authorization: ${API_KEY}" \ - --data "id=${TEST_ORG_ID}&object=test_org_member&object_type=user&capacity=member" \ + --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \ ${CKAN_ACTION_URL}/member_create ## # END. @@ -79,4 +74,4 @@ curl -LsH "Authorization: ${API_KEY}" \ "author_email": "admin@localhost", "license_id": "other-open", "notes": "test"}' \ ${CKAN_ACTION_URL}/package_create -. ${APP_DIR}/scripts/deactivate +. ${APP_DIR}/bin/deactivate diff --git a/.docker/scripts/deactivate b/bin/deactivate similarity index 100% rename from .docker/scripts/deactivate rename to bin/deactivate diff --git a/.docker/scripts/doctor.sh b/bin/doctor.sh similarity index 100% rename from .docker/scripts/doctor.sh rename to bin/doctor.sh diff --git a/.docker/scripts/extract-id.py b/bin/extract-id.py similarity index 100% rename from .docker/scripts/extract-id.py rename to bin/extract-id.py diff --git a/.docker/scripts/init-ext.sh b/bin/init-ext.sh similarity index 94% rename from .docker/scripts/init-ext.sh rename to bin/init-ext.sh index f32ececb..aee0e156 100755 --- a/.docker/scripts/init-ext.sh +++ b/bin/init-ext.sh @@ -2,7 +2,7 @@ ## # Install current extension. # -set -e +set -ex install_requirements () { PROJECT_DIR=$1 @@ -26,7 +26,7 @@ install_requirements () { done } -. ${APP_DIR}/scripts/activate +. ${APP_DIR}/bin/activate if [ "$CKAN_VERSION" = "2.9-py2" ]; then @@ -45,4 +45,4 @@ installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1 # Validate that the extension was installed correctly. if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi -. ${APP_DIR}/scripts/deactivate +. ${APP_DIR}/bin/deactivate diff --git a/.docker/scripts/init.sh b/bin/init.sh similarity index 79% rename from .docker/scripts/init.sh rename to bin/init.sh index a9736f01..1b1d213b 100755 --- a/.docker/scripts/init.sh +++ b/bin/init.sh @@ -4,7 +4,7 @@ # set -e -. ${APP_DIR}/scripts/activate +. ${APP_DIR}/bin/activate CLICK_ARGS="--yes" ckan_cli db clean ckan_cli db init ckan_cli db upgrade @@ -13,4 +13,4 @@ ckan_cli db upgrade PASTER_PLUGIN=ckanext-validation ckan_cli validation init-db # Create some base test data -. $APP_DIR/scripts/create-test-data.sh +. $APP_DIR/bin/create-test-data.sh diff --git a/.circleci/process-artifacts.sh b/bin/process-artifacts.sh similarity index 100% rename from .circleci/process-artifacts.sh rename to bin/process-artifacts.sh diff --git a/.docker/scripts/serve.sh b/bin/serve.sh similarity index 89% rename from .docker/scripts/serve.sh rename to bin/serve.sh index e68b41c5..af0d8249 100755 --- a/.docker/scripts/serve.sh +++ b/bin/serve.sh @@ -5,7 +5,7 @@ dockerize -wait tcp://postgres:5432 -timeout 1m dockerize -wait tcp://solr:8983 -timeout 1m dockerize -wait tcp://redis:6379 -timeout 1m -. ${APP_DIR}/scripts/activate +. ${APP_DIR}/bin/activate if (which ckan > /dev/null); then ckan -c ${CKAN_INI} run -r else diff --git a/.circleci/test.sh b/bin/test.sh similarity index 83% rename from .circleci/test.sh rename to bin/test.sh index a2d91da7..f361aacf 100755 --- a/.circleci/test.sh +++ b/bin/test.sh @@ -12,4 +12,5 @@ ahoy test-unit echo "==> Run BDD tests" ahoy install-site +ahoy cli "rm -r test/screenshots || true" ahoy test-bdd || (ahoy logs; exit 1) diff --git a/ckanext/validation/assets/resource.config b/ckanext/validation/assets/resource.config deleted file mode 100644 index ecea98a6..00000000 --- a/ckanext/validation/assets/resource.config +++ /dev/null @@ -1,25 +0,0 @@ -[groups] - -main = - css/validation.css - -resource-schema-form = - js/module-resource-schema.js - -report-form = - vendor/goodtables-ui/goodtables-ui.css - vendor/goodtables-ui/goodtables-ui.js - - css/validation-report.css - css/validation-report-form.css - - js/module-validation-report.js - js/module-modal-dialog.js - -report = - vendor/goodtables-ui/goodtables-ui.css - vendor/goodtables-ui/goodtables-ui.js - - css/validation-report.css - - js/module-validation-report.js diff --git a/ckanext/validation/helpers.py b/ckanext/validation/helpers.py index 76895f61..ddb83c13 100644 --- a/ckanext/validation/helpers.py +++ b/ckanext/validation/helpers.py @@ -22,7 +22,9 @@ def _get_helpers(): def get_validation_badge(resource, in_listing=False): afterDate = config.get('ckanext.validation.show_badges_after_last_modified_date', "") - if afterDate and h.date_str_to_datetime(afterDate) >= h.date_str_to_datetime(resource['last_modified']): + if afterDate and (not resource.get('last_modified') + or h.date_str_to_datetime(afterDate) + >= h.date_str_to_datetime(resource['last_modified'])): return '' if in_listing and not asbool( diff --git a/ckanext/validation/templates/package/resource_read.html b/ckanext/validation/templates/package/resource_read.html index 5849d13d..5ff88055 100644 --- a/ckanext/validation/templates/package/resource_read.html +++ b/ckanext/validation/templates/package/resource_read.html @@ -6,8 +6,7 @@

{{ h.resource_display_name(res) | truncate(50) }} {{ h.get_validation_badge(res)|safe }}

- {% set type = 'asset' %} - {% include 'validation/snippets/validation_style_' ~ type ~ '.html' %} + {% asset 'ckanext-validation/main-css' %} {% endblock %} diff --git a/ckanext/validation/templates/package/snippets/resource_item.html b/ckanext/validation/templates/package/snippets/resource_item.html index 4377863c..d5baef7c 100644 --- a/ckanext/validation/templates/package/snippets/resource_item.html +++ b/ckanext/validation/templates/package/snippets/resource_item.html @@ -4,8 +4,6 @@ {{ super() }} {{ h.get_validation_badge(res, in_listing=True)|safe }} -{% set type = 'asset' %} -{% include 'validation/snippets/validation_style_' ~ type ~ '.html' %} -{% endblock %} - + {% asset 'ckanext-validation/main-css' %} +{% endblock %} diff --git a/ckanext/validation/templates/scheming/form_snippets/resource_schema.html b/ckanext/validation/templates/scheming/form_snippets/resource_schema.html index 4f4c2c99..c69d6bf0 100644 --- a/ckanext/validation/templates/scheming/form_snippets/resource_schema.html +++ b/ckanext/validation/templates/scheming/form_snippets/resource_schema.html @@ -64,7 +64,6 @@ {% set existing_value = h.scheming_display_json_value(value, indent=None) if is_json else value %} - {% set type = 'asset' %} - {% include 'validation/snippets/validation_resource-schema-form_' ~ type ~ '.html' %} + {% asset 'ckanext-validation/resource-schema-form' %} diff --git a/ckanext/validation/templates/validation/snippets/validation_report_asset.html b/ckanext/validation/templates/validation/snippets/validation_report_asset.html deleted file mode 100644 index 987d4e97..00000000 --- a/ckanext/validation/templates/validation/snippets/validation_report_asset.html +++ /dev/null @@ -1,3 +0,0 @@ -{% asset 'ckanext-validation/goodtables-ui' %} -{% asset 'ckanext-validation/validation-report-css' %} -{% asset 'ckanext-validation/module-validation-report' %} diff --git a/ckanext/validation/templates/validation/snippets/validation_report_dialog.html b/ckanext/validation/templates/validation/snippets/validation_report_dialog.html index de6bcd01..33f64e6d 100644 --- a/ckanext/validation/templates/validation/snippets/validation_report_dialog.html +++ b/ckanext/validation/templates/validation/snippets/validation_report_dialog.html @@ -14,5 +14,8 @@

-{% set type = 'asset' %} -{% include 'validation/snippets/validation_report_form_' ~ type ~ '.html' %} +{% asset 'ckanext-validation/goodtables-ui' %} +{% asset 'ckanext-validation/validation-report-css' %} +{% asset 'ckanext-validation/module-validation-report' %} +{% asset 'ckanext-validation/validation-report-form-css' %} +{% asset 'ckanext-validation/module-modal-dialog' %} diff --git a/ckanext/validation/templates/validation/snippets/validation_report_dialog_bs2.html b/ckanext/validation/templates/validation/snippets/validation_report_dialog_bs2.html index 5a980fab..17d9cf73 100644 --- a/ckanext/validation/templates/validation/snippets/validation_report_dialog_bs2.html +++ b/ckanext/validation/templates/validation/snippets/validation_report_dialog_bs2.html @@ -7,7 +7,8 @@

{{ _('Data Validation Report') }}

- -{% set type = 'asset' %} -{% include 'validation/snippets/validation_report_form_' ~ type ~ '.html' %} - +{% asset 'ckanext-validation/goodtables-ui' %} +{% asset 'ckanext-validation/validation-report-css' %} +{% asset 'ckanext-validation/module-validation-report' %} +{% asset 'ckanext-validation/validation-report-form-css' %} +{% asset 'ckanext-validation/module-modal-dialog' %} diff --git a/ckanext/validation/templates/validation/snippets/validation_report_form_asset.html b/ckanext/validation/templates/validation/snippets/validation_report_form_asset.html deleted file mode 100644 index de400faf..00000000 --- a/ckanext/validation/templates/validation/snippets/validation_report_form_asset.html +++ /dev/null @@ -1,5 +0,0 @@ -{% asset 'ckanext-validation/goodtables-ui' %} -{% asset 'ckanext-validation/validation-report-css' %} -{% asset 'ckanext-validation/validation-report-form-css' %} -{% asset 'ckanext-validation/module-validation-report' %} -{% asset 'ckanext-validation/module-modal-dialog' %} \ No newline at end of file diff --git a/ckanext/validation/templates/validation/snippets/validation_resource-schema-form_asset.html b/ckanext/validation/templates/validation/snippets/validation_resource-schema-form_asset.html deleted file mode 100644 index 62bb7b47..00000000 --- a/ckanext/validation/templates/validation/snippets/validation_resource-schema-form_asset.html +++ /dev/null @@ -1 +0,0 @@ -{% asset 'ckanext-validation/resource-schema-form' %} \ No newline at end of file diff --git a/ckanext/validation/templates/validation/snippets/validation_style_asset.html b/ckanext/validation/templates/validation/snippets/validation_style_asset.html deleted file mode 100644 index d871253b..00000000 --- a/ckanext/validation/templates/validation/snippets/validation_style_asset.html +++ /dev/null @@ -1 +0,0 @@ -{% asset 'ckanext-validation/main-css' %} \ No newline at end of file diff --git a/ckanext/validation/templates/validation/validation_read.html b/ckanext/validation/templates/validation/validation_read.html index d841e096..a27ce8bd 100644 --- a/ckanext/validation/templates/validation/validation_read.html +++ b/ckanext/validation/templates/validation/validation_read.html @@ -1,9 +1,7 @@ -{% set type = 'asset' %} - -{% include 'validation/snippets/validation_style_' ~ type ~ '.html' %} - {% extends "package/base.html" %} +{% asset 'ckanext-validation/main-css' %} + {%- block subtitle %}{{ _('Validation Report') }}{% endblock -%} {% block breadcrumb_content_selected %}{% endblock %} @@ -40,9 +38,9 @@

{{ h.resource_display_name(resource) | truncate(50) }} - - {% include 'validation/snippets/validation_report_' ~ type ~ '.html' %} - + {% asset 'ckanext-validation/goodtables-ui' %} + {% asset 'ckanext-validation/validation-report-css' %} + {% asset 'ckanext-validation/module-validation-report' %} {% endblock %} diff --git a/ckanext/validation/tests/test_logic.py b/ckanext/validation/tests/test_logic.py index d5946d27..1688c39d 100644 --- a/ckanext/validation/tests/test_logic.py +++ b/ckanext/validation/tests/test_logic.py @@ -495,10 +495,10 @@ def test_package_patch_with_resources_does_not_set_context_flag(self): @pytest.mark.usefixtures("clean_db", "validation_setup") class TestAuth(object): + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_run_anon(self): - resource = factories.Resource() - context = {'user': None, 'model': model} with pytest.raises(tk.NotAuthorized): @@ -506,22 +506,24 @@ def test_run_anon(self): context=context, resource_id=resource['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_run_sysadmin(self): resource = factories.Resource() sysadmin = factories.Sysadmin() - context = {'user': sysadmin['name'], 'model': model} assert call_auth('resource_validation_run', context=context, resource_id=resource['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_run_non_auth_user(self): user = factories.User() org = factories.Organization() dataset = factories.Dataset(owner_org=org['id'], resources=[factories.Resource()]) - context = {'user': user['name'], 'model': model} with pytest.raises(tk.NotAuthorized): @@ -529,6 +531,8 @@ def test_run_non_auth_user(self): context=context, resource_id=dataset['resources'][0]['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_run_auth_user(self): user = factories.User() org = factories.Organization(users=[{ @@ -537,16 +541,16 @@ def test_run_auth_user(self): }]) dataset = factories.Dataset(owner_org=org['id'], resources=[factories.Resource()]) - context = {'user': user['name'], 'model': model} assert call_auth('resource_validation_run', context=context, resource_id=dataset['resources'][0]['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_delete_anon(self): resource = factories.Resource() - context = {'user': None, 'model': model} with pytest.raises(tk.NotAuthorized): @@ -554,22 +558,24 @@ def test_delete_anon(self): context=context, resource_id=resource['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_delete_sysadmin(self): resource = factories.Resource() sysadmin = factories.Sysadmin() - context = {'user': sysadmin['name'], 'model': model} assert call_auth('resource_validation_delete', context=context, resource_id=resource['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_delete_non_auth_user(self): user = factories.User() org = factories.Organization() dataset = factories.Dataset(owner_org=org['id'], resources=[factories.Resource()]) - context = {'user': user['name'], 'model': model} with pytest.raises(tk.NotAuthorized): @@ -577,6 +583,8 @@ def test_delete_non_auth_user(self): context=context, resource_id=dataset['resources'][0]['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_delete_auth_user(self): user = factories.User() org = factories.Organization(users=[{ @@ -585,42 +593,44 @@ def test_delete_auth_user(self): }]) dataset = factories.Dataset(owner_org=org['id'], resources=[factories.Resource()]) - context = {'user': user['name'], 'model': model} assert call_auth('resource_validation_delete', context=context, resource_id=dataset['resources'][0]['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_show_anon(self): resource = factories.Resource() - context = {'user': None, 'model': model} assert call_auth('resource_validation_show', context=context, resource_id=resource['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_show_anon_public_dataset(self): user = factories.User() org = factories.Organization() dataset = factories.Dataset(owner_org=org['id'], resources=[factories.Resource()], private=False) - context = {'user': user['name'], 'model': model} assert call_auth('resource_validation_show', context=context, resource_id=dataset['resources'][0]['id']) + @pytest.mark.ckan_config("ckanext.validation.run_on_create_sync", False) + @pytest.mark.ckan_config("ckanext.validation.run_on_update_sync", False) def test_show_anon_private_dataset(self): user = factories.User() org = factories.Organization() dataset = factories.Dataset(owner_org=org['id'], resources=[factories.Resource()], private=True) - context = {'user': user['name'], 'model': model} with pytest.raises(tk.NotAuthorized): diff --git a/test/features/environment.py b/test/features/environment.py index 09c59658..73e588b3 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -14,7 +14,7 @@ # URL of remote Chrome instance. REMOTE_CHROME_URL = 'http://chrome:4444/wd/hub' -# @see .docker/scripts/init.sh for credentials. +# @see bin/init.sh for credentials. PERSONAS = { 'SysAdmin': { 'name': u'admin', diff --git a/test/features/resource_validation.feature b/test/features/resource_validation.feature index 02bd342e..010b3b3e 100644 --- a/test/features/resource_validation.feature +++ b/test/features/resource_validation.feature @@ -4,8 +4,8 @@ Feature: Resource validation Given "TestOrgEditor" as the persona When I log in And I open the new resource form for dataset "warandpeace" - And I fill in "name" with "Test validation schema" And I attach the file "test.csv" to "upload" + And I fill in "name" with "Test validation schema" And I fill in "description" with "Testing validation schema" And I attach the file "test_schema.json" to "schema_upload" And I press the element with xpath "//button[contains(@class, 'btn-primary')]" @@ -35,8 +35,8 @@ Feature: Resource validation Given "TestOrgEditor" as the persona When I log in And I open the new resource form for dataset "warandpeace" - And I fill in "name" with "Test validation options" And I attach the file "test.csv" to "upload" + And I fill in "name" with "Test validation options" And I fill in "description" with "Testing validation options" And I attach the file "test_schema.json" to "schema_upload" And I fill in "validation_options" with "{"headers": 1}" @@ -63,8 +63,8 @@ Feature: Resource validation Given "TestOrgEditor" as the persona When I log in And I open the new resource form for dataset "warandpeace" - And I fill in "name" with "Test valid CSV create" And I attach the file "test.csv" to "upload" + And I fill in "name" with "Test valid CSV create" And I attach the file "test_schema.json" to "schema_upload" And I fill in "description" with "Testing validation that should pass" And I execute the script "document.getElementById('field-format').value='CSV'" @@ -80,8 +80,8 @@ Feature: Resource validation Given "TestOrgEditor" as the persona When I log in And I open the new resource form for dataset "warandpeace" - And I fill in "name" with "Test valid CSV update" And I attach the file "invalid.csv" to "upload" + And I fill in "name" with "Test valid CSV update" And I fill in "description" with "Testing validation that should pass on update" And I press the element with xpath "//button[contains(@class, 'btn-primary')]" Then I should see "Test valid CSV update" @@ -101,8 +101,8 @@ Feature: Resource validation Given "TestOrgEditor" as the persona When I log in And I open the new resource form for dataset "warandpeace" - And I fill in "name" with "Test invalid CSV update" And I attach the file "test.csv" to "upload" + And I fill in "name" with "Test invalid CSV update" And I attach the file "test_schema.json" to "schema_upload" And I fill in "description" with "Testing validation that should fail on update" And I press the element with xpath "//button[contains(@class, 'btn-primary')]" diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py index c789500f..9950d0c6 100644 --- a/test/features/steps/steps.py +++ b/test/features/steps/steps.py @@ -53,8 +53,7 @@ def attempt_login(context, password): @step(u'I open the new resource form for dataset "{name}"') def go_to_new_resource_form(context, name): context.execute_steps(u""" - When I go to dataset "{name}" - And I click the link with text that contains "Manage" + When I edit the "{name}" dataset And I click the link with text that contains "Resources" And I click the link with text that contains "Add new resource" """.format(name=name)) @@ -65,11 +64,12 @@ def add_resource(context, name, url): context.execute_steps(u""" When I log in And I open the new resource form for dataset "warandpeace" - And I press the element with xpath "//form[@id='resource-edit']//a[string() = 'Link']" - And I fill in "name" with "{}" - And I fill in "url" with "{}" - And I press the element with xpath "//button[contains(string(), 'Add')]" - """.format(name, url)) + And I execute the script "$('#resource-edit [name=url]').val('{url}')" + And I fill in "name" with "{name}" + And I fill in "description" with "description" + And I execute the script "document.getElementById('field-format').value='HTML'" + And I press the element with xpath "//form[contains(@class, 'resource-form')]//button[contains(@class, 'btn-primary')]" + """.format(name=name, url=url)) @step(u'I go to dataset page') @@ -82,6 +82,14 @@ def go_to_dataset(context, name): when_i_visit_url(context, '/dataset/' + name) +@step(u'I edit the "{name}" dataset') +def edit_dataset(context, name): + context.execute_steps(u""" + When I go to dataset "{name}" + And I click the link with text that contains "Manage" + """.format(name=name)) + + @step(u'I go to organisation page') def go_to_organisation_page(context): when_i_visit_url(context, '/organization') @@ -109,7 +117,16 @@ def go_to_user_profile(context, user_id): @step(u'I go to the dashboard') def go_to_dashboard(context): - when_i_visit_url(context, '/dashboard') + context.execute_steps(u""" + When I visit "/dashboard/datasets" + """) + + +@step(u'I should see my datasets') +def dashboard_datasets(context): + context.execute_steps(u""" + Then I should see an element with xpath "//li[contains(@class, 'active') and contains(string(), 'My Datasets')]" + """) @step(u'I go to the "{user_id}" user API') @@ -119,12 +136,19 @@ def go_to_user_show(context, user_id): @step(u'I view the "{group_id}" group API "{including}" users') def go_to_group_including_users(context, group_id, including): - when_i_visit_url(context, r'/api/3/action/group_show?id={}&include_users={}'.format(group_id, including in ['with', 'including'])) + when_i_visit_url( + context, r'/api/3/action/group_show?id={}&include_users={}'.format( + group_id, including in ['with', 'including'])) @step(u'I view the "{organisation_id}" organisation API "{including}" users') def go_to_organisation_including_users(context, organisation_id, including): - when_i_visit_url(context, r'/api/3/action/organization_show?id={}&include_users={}'.format(organisation_id, including in ['with', 'including'])) + when_i_visit_url( + context, r'/api/3/action/organization_show?id={}&include_users={}'.format( + organisation_id, including in ['with', 'including'])) + + +# ckanext-validation @step(u'I should see a validation timestamp')