From fc3b6cd60a6b7365f2ca5c76bc64be44e788448e Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 22 Mar 2023 11:50:53 +1000 Subject: [PATCH 1/9] [QOLDEV-323] update dependencies to handle CKAN 2.10 - Update ckanext-scheming to use 'unicode_safe' validator instead of 'unicode' - Use newer 'Faker' lib when testing on Python 3+ --- dev-requirements.txt | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3cbf89f0..5b65fae2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,4 +9,4 @@ pytest-ckan pytest-cov selenium==3.141.0 splinter>=0.13.0,<0.17 -faker==3.0.1 +faker==18.3.0 diff --git a/requirements.txt b/requirements.txt index c03514b9..ccb14fba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ ckantoolkit>=0.0.3 goodtables==1.5.1 six>=1.13.0 --e git+https://github.com/ckan/ckanext-scheming.git@release-2.1.0#egg=ckanext-scheming +-e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming From 5a44439e653d4e6180a7eea4ecc74af9053ff2a9 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 22 Mar 2023 12:08:07 +1000 Subject: [PATCH 2/9] [QOLDEV-323] generate test data via API rather than CLI - CLI interface is changing in CKAN 2.10, API is more stable --- .docker/scripts/create-test-data.sh | 80 ++--------------------------- test/features/environment.py | 15 ------ 2 files changed, 4 insertions(+), 91 deletions(-) diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index 74dacd75..1eb43d50 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -73,82 +73,10 @@ curl -LsH "Authorization: ${API_KEY}" \ # END. # -# Creating test data hierarchy which creates organisations assigned to datasets -ckan_cli create-test-data hierarchy - # Creating basic test data which has datasets with resources -ckan_cli create-test-data basic - -add_user_if_needed organisation_admin "Organisation Admin" organisation_admin@localhost -add_user_if_needed editor "Publisher" publisher@localhost -add_user_if_needed foodie "Foodie" foodie@localhost -add_user_if_needed group_admin "Group Admin" group_admin@localhost -add_user_if_needed walker "Walker" walker@localhost - -echo "Assigning test Datasets to Organisation..." - -echo "Updating annakarenina to use ${TEST_ORG_TITLE} organisation:" -package_owner_org_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "id=annakarenina&organization_id=${TEST_ORG_NAME}" \ - ${CKAN_ACTION_URL}/package_owner_org_update -) -echo ${package_owner_org_update} - -echo "Updating warandpeace to use ${TEST_ORG_TITLE} organisation:" -package_owner_org_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "id=warandpeace&organization_id=${TEST_ORG_NAME}" \ - ${CKAN_ACTION_URL}/package_owner_org_update -) -echo ${package_owner_org_update} - -echo "Updating organisation_admin to have admin privileges in the department-of-health Organisation:" -organisation_admin_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "id=department-of-health&username=organisation_admin&role=admin" \ - ${CKAN_ACTION_URL}/organization_member_create -) -echo ${organisation_admin_update} - -echo "Updating publisher to have editor privileges in the department-of-health Organisation:" -publisher_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "id=department-of-health&username=editor&role=editor" \ - ${CKAN_ACTION_URL}/organization_member_create -) -echo ${publisher_update} - -echo "Updating foodie to have admin privileges in the food-standards-agency Organisation:" -foodie_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "id=food-standards-agency&username=foodie&role=admin" \ - ${CKAN_ACTION_URL}/organization_member_create -) -echo ${foodie_update} - -echo "Creating non-organisation group:" -group_create=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "name=silly-walks" \ - ${CKAN_ACTION_URL}/group_create -) -echo ${group_create} - -echo "Updating group_admin to have admin privileges in the silly-walks group:" -group_admin_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "id=silly-walks&username=group_admin&role=admin" \ - ${CKAN_ACTION_URL}/group_member_create -) -echo ${group_admin_update} - -echo "Updating walker to have editor privileges in the silly-walks group:" -walker_update=$( \ - curl -LsH "Authorization: ${API_KEY}" \ - --data "id=silly-walks&username=walker&role=editor" \ - ${CKAN_ACTION_URL}/group_member_create -) -echo ${walker_update} +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"name": "warandpeace", "owner_org": "'"${TEST_ORG_ID}"'", +"author_email": "admin@localhost", "license_id": "other-open", "notes": "test"}' \ + ${CKAN_ACTION_URL}/package_create . ${APP_DIR}/scripts/deactivate diff --git a/test/features/environment.py b/test/features/environment.py index 0c9b4c0b..09c59658 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -36,21 +36,6 @@ 'email': u'group_admin@localhost', 'password': u'Password123!' }, - 'Publisher': { - 'name': u'editor', - 'email': u'publisher@localhost', - 'password': u'Password123!' - }, - 'Walker': { - 'name': u'walker', - 'email': u'walker@localhost', - 'password': u'Password123!' - }, - 'Foodie': { - 'name': u'foodie', - 'email': u'foodie@localhost', - 'password': u'Password123!' - }, 'TestOrgAdmin': { 'name': u'test_org_admin', 'email': u'test_org_admin@localhost', From 0379816d07ca85d861796ab97db6ed9d2384be74 Mon Sep 17 00:00:00 2001 From: william dutton Date: Wed, 22 Mar 2023 13:43:46 +1000 Subject: [PATCH 3/9] QOLSVC-1251 Add flag to hide if before a date --- ckanext/validation/helpers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ckanext/validation/helpers.py b/ckanext/validation/helpers.py index e2bb32c4..593c6ed4 100644 --- a/ckanext/validation/helpers.py +++ b/ckanext/validation/helpers.py @@ -4,7 +4,7 @@ from six.moves.urllib.parse import urlparse from six import string_types from ckantoolkit import url_for, _, config, asbool,\ - literal, check_ckan_version + literal, check_ckan_version, h def _get_helpers(): @@ -23,6 +23,10 @@ def _get_helpers(): def get_validation_badge(resource, in_listing=False): + afterDate = config.get('ckanext.validation.show_badges_after_last_modified_Date', "") + if afterDate != "" and h.date_str_to_datetime(afterDate) >= h.date_str_to_datetime(resource['last_modified']) : + return '' + if in_listing and not asbool( config.get('ckanext.validation.show_badges_in_listings', True)): return '' From 446122e0633e1f254e46405b81bf8475037db919 Mon Sep 17 00:00:00 2001 From: william dutton Date: Wed, 22 Mar 2023 13:44:00 +1000 Subject: [PATCH 4/9] update success to valid to match other status types --- ckanext/validation/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/validation/helpers.py b/ckanext/validation/helpers.py index 593c6ed4..3a99a0c4 100644 --- a/ckanext/validation/helpers.py +++ b/ckanext/validation/helpers.py @@ -24,7 +24,7 @@ def _get_helpers(): def get_validation_badge(resource, in_listing=False): afterDate = config.get('ckanext.validation.show_badges_after_last_modified_Date', "") - if afterDate != "" and h.date_str_to_datetime(afterDate) >= h.date_str_to_datetime(resource['last_modified']) : + if afterDate != "" and h.date_str_to_datetime(afterDate) >= h.date_str_to_datetime(resource['last_modified']): return '' if in_listing and not asbool( @@ -35,7 +35,7 @@ def get_validation_badge(resource, in_listing=False): return '' statuses = { - 'success': _('success'), + 'success': _('valid'), 'failure': _('failure'), 'invalid': _('invalid'), 'error': _('error'), From a7a6ea4973640323aa2f71b5baac52f3f7d43d67 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 22 Mar 2023 13:53:10 +1000 Subject: [PATCH 5/9] [QOLDEV-323] update test config to permit unowned datasets - This is the default in CKAN 2.9, but not 2.10 --- .docker/test.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.docker/test.ini b/.docker/test.ini index c2d29f97..ad0f79b2 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -56,6 +56,8 @@ ckan.datastore.default_fts_index_method = gist ## Site Settings. ckan.site_url = http://ckan:5000/ +ckan.auth.create_unowned_dataset=true + ## Search Settings ckan.site_id = default From 10719e2e4c027dedd67f8d7b802a07f9715d7a48 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 22 Mar 2023 13:54:05 +1000 Subject: [PATCH 6/9] [QOLDEV-323] adjust test monkey-patch to be more stable - Target a function, which will remain in CKAN 2.10, rather than an internal variable --- ckanext/validation/tests/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/validation/tests/fixtures.py b/ckanext/validation/tests/fixtures.py index c59ae59d..a6747803 100644 --- a/ckanext/validation/tests/fixtures.py +++ b/ckanext/validation/tests/fixtures.py @@ -20,7 +20,7 @@ @pytest.fixture def validation_setup(monkeypatch, ckan_config, tmpdir): monkeypatch.setitem(ckan_config, u'ckan.storage_path', str(tmpdir)) - monkeypatch.setattr(uploader, u'_storage_path', str(tmpdir)) + monkeypatch.setattr(uploader, u'get_storage_path', lambda: str(tmpdir)) if not tables_exist(): create_tables() From dc84f6d43abc45c721c4c0551dbe8f40c34efabc Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 22 Mar 2023 14:21:32 +1000 Subject: [PATCH 7/9] [QOLDEV-323] replace obsolete mock assertions with newer syntax - Use 'mock_obj.assert_not_called()' instead of 'assert mock_obj.not_called()' --- ckanext/validation/tests/test_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/validation/tests/test_plugin.py b/ckanext/validation/tests/test_plugin.py index 986f02ff..758e7e08 100644 --- a/ckanext/validation/tests/test_plugin.py +++ b/ckanext/validation/tests/test_plugin.py @@ -95,7 +95,7 @@ def test_validation_run_on_format_change(self, mock_enqueue, """Validation must be triggered during update on changing format""" resource = resource_factory(format="PDF") - assert mock_enqueue.not_called() + mock_enqueue.assert_not_called() resource['format'] = 'CSV' @@ -109,7 +109,7 @@ def test_validation_run_on_validation_options_change( validation_options""" resource = resource_factory(format="PDF") - assert mock_enqueue.not_called() + mock_enqueue.assert_not_called() resource['validation_options'] = {'headers': 1, 'skip_rows': ['#']} resource['format'] = 'CSV' From cc0f2e98ad9aed5326510e928c4622bf6c7a3e9a Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 22 Mar 2023 14:47:52 +1000 Subject: [PATCH 8/9] [QOLDEV-323] update Selenium and Splinter when on Python 3 --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 5b65fae2..c2e6f1dc 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,6 +7,6 @@ mock pyfakefs==2.7 pytest-ckan pytest-cov -selenium==3.141.0 -splinter>=0.13.0,<0.17 +selenium==4.8.2 +splinter>=0.13.0 faker==18.3.0 From ad320d302a4cbd96ff40b21824bd06889e8e7105 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Wed, 22 Mar 2023 16:43:20 +1000 Subject: [PATCH 9/9] [QOLSVC-1251] fix config key capitalisation --- ckanext/validation/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/validation/helpers.py b/ckanext/validation/helpers.py index 3a99a0c4..91e0ed2b 100644 --- a/ckanext/validation/helpers.py +++ b/ckanext/validation/helpers.py @@ -23,8 +23,8 @@ def _get_helpers(): def get_validation_badge(resource, in_listing=False): - afterDate = config.get('ckanext.validation.show_badges_after_last_modified_Date', "") - if afterDate != "" and h.date_str_to_datetime(afterDate) >= h.date_str_to_datetime(resource['last_modified']): + afterDate = config.get('ckanext.validation.show_badges_after_last_modified_date', "") + if afterDate and h.date_str_to_datetime(afterDate) >= h.date_str_to_datetime(resource['last_modified']): return '' if in_listing and not asbool(