From c9b1f6929f53dc7f57c3c17749891089282bbddb Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 16 Nov 2023 14:05:05 +0100 Subject: [PATCH 1/4] parse horizontalService from main service bucket --- modules/parse/base.py | 1 + modules/parse/provider_topology.py | 6 +++++- modules/tasks/provider_topology.py | 6 +++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/parse/base.py b/modules/parse/base.py index ab24a46b..a408e28d 100644 --- a/modules/parse/base.py +++ b/modules/parse/base.py @@ -122,6 +122,7 @@ def csv_to_json(self, data): continue num_item = 0 datum = {} + for item in header: datum[item] = row[num_item] num_item = num_item + 1 diff --git a/modules/parse/provider_topology.py b/modules/parse/provider_topology.py index b88b38b5..d704de2f 100644 --- a/modules/parse/provider_topology.py +++ b/modules/parse/provider_topology.py @@ -22,7 +22,7 @@ def build_urlpath_id(http_endpoint): return uuid.uuid3(uuid.NAMESPACE_URL, path) else: return None - + class ParseResources(ParseHelpers): def __init__(self, logger, data=None, keys=[], custname=None): @@ -48,6 +48,10 @@ def _parse_data(self): key_true = extras.get(key, False) if key_true: tags.append(key) + for key in self._keys: + key_true = resource.get(key, False) + if key_true: + tags.append(key) if not resource.get('name', False): continue self._resources.append({ diff --git a/modules/tasks/provider_topology.py b/modules/tasks/provider_topology.py index 54a81970..a03ae672 100644 --- a/modules/tasks/provider_topology.py +++ b/modules/tasks/provider_topology.py @@ -42,7 +42,7 @@ def _parse(self): total = doc['total'] from_index = doc['from'] to_index = doc['to'] - + return total, from_index, to_index @@ -88,7 +88,7 @@ def parse_source_extensions(self, extensions, groupnames): resources_extended = ParseExtensions(self.logger, extensions, groupnames, self.uidservendp, self.logger.customer) return resources_extended.get_extensions() - + def parse_source_topo(self, resources, providers): topo = ParseTopo(self.logger, providers, resources, self.uidservendp, self.logger.customer) @@ -167,7 +167,7 @@ async def fetch_data(self, feed, access_token, paginated): remote_topo.path, from_index, num), - headers=headers) + headers=headers) await session.close() return res From f8e71674154d1e5ad1569fff90b9fef08401e2af Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 16 Nov 2023 17:04:31 +0100 Subject: [PATCH 2/4] add sample provider and resource with horizontalService flag under main service bucket --- tests/sample-private-provider.json | 95 +++++++++++++++++ tests/sample-private-resource.json | 166 +++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+) diff --git a/tests/sample-private-provider.json b/tests/sample-private-provider.json index 0ae96b8a..0812b06e 100644 --- a/tests/sample-private-provider.json +++ b/tests/sample-private-provider.json @@ -248,6 +248,101 @@ ] }, "id": "openaire" + }, + { + "metadata": { + "registeredBy": "Sxxxxx Uxxxx", + "registeredAt": "1626436442264", + "modifiedBy": "Axxxxxxx Gxxxxxxxxx", + "modifiedAt": "1655453425739", + "terms": [ + "sxxxxx.uxxxx@xxx.xx", + "axxxxxxx.gxxxxxxxxx@xxxxxxxx.xx" + ], + "published": false + }, + "active": true, + "suspended": false, + "identifiers": null, + "migrationStatus": null, + "loggingInfo": null, + "latestAuditInfo": null, + "latestOnboardingInfo": null, + "latestUpdateInfo": null, + "status": "approved provider", + "templateStatus": "approved template", + "provider": { + "id": "riga_stradins_university", + "abbreviation": "RIga Stradins University", + "name": "Riga Stradins University", + "website": "https://www.rsu.lv/", + "legalEntity": true, + "legalStatus": "provider_legal_status-public_legal_entity", + "hostingLegalEntity": null, + "alternativeIdentifiers": null, + "description": "Riga Stradins University from Riga, Latvia is one of the leading and most advanced higher education teaching institutions in the Baltic region. The RSU research and teaching staff are professionals in their areas of expertise and are opinion leaders in the society. RSUs mission is to train highly qualified experts in the field of healthcare and social sciences, so they can serve the society of Latvia, the European Union, as well as the world. ", + "logo": "https://www.rsu.lv/sites/default/files/imce/rsu_alternate_logo_en.png", + "multimedia": [], + "scientificDomains": [ + { + "scientificDomain": "scientific_domain-medical_and_health_sciences", + "scientificSubdomain": "scientific_subdomain-medical_and_health_sciences-health_sciences" + } + ], + "tags": [ + "university", + "Latvia", + "Health", + "Medicine" + ], + "structureTypes": [], + "location": { + "streetNameAndNumber": "16 Dzirciema Street", + "postalCode": " LV-1007", + "city": "Riga", + "region": "Riga", + "country": "LV" + }, + "mainContact": { + "firstName": "Ixxxxxx", + "lastName": "Kxxxxxxxxx", + "email": "Ixxxxxx.Kxxxxxxxxx@rsu.lv", + "phone": "+37167455586", + "position": "Science project coordinator" + }, + "publicContacts": [ + { + "firstName": "Ixxxxxx", + "lastName": "Kxxxxxxxxx", + "email": "Ixxxxxx.Kxxxxxxxxx@xxx.xx", + "phone": "+xxxxxxxxxxx", + "position": "Science project coordinator" + } + ], + "lifeCycleStatus": "provider_life_cycle_status-operational", + "certifications": [], + "participatingCountries": [ + "LV" + ], + "affiliations": [], + "networks": [], + "catalogueId": "eosc", + "esfriDomains": [], + "esfriType": null, + "merilScientificDomains": [], + "areasOfActivity": [], + "societalGrandChallenges": [], + "nationalRoadmaps": [], + "users": [ + { + "id": "", + "email": "Ixxxxxx.Kxxxxxxxxx@xxx.xx", + "name": "Ixxxxxx", + "surname": "Kxxxxxxxxx" + } + ] + }, + "id": "riga_stradins_university" } ] } diff --git a/tests/sample-private-resource.json b/tests/sample-private-resource.json index 1fc74157..61530aef 100644 --- a/tests/sample-private-resource.json +++ b/tests/sample-private-resource.json @@ -650,6 +650,172 @@ "pricing": null }, "id": "openaire.validator" + }, + { + "metadata": { + "registeredBy": "Ixxxxxx Kxxxxxxxxx", + "registeredAt": "1652777979850", + "modifiedBy": "Ixxxxxx Kxxxxxxxxx", + "modifiedAt": "1652943494220", + "terms": null, + "published": false + }, + "active": true, + "suspended": false, + "identifiers": null, + "migrationStatus": null, + "loggingInfo": null, + "latestAuditInfo": null, + "latestOnboardingInfo": null, + "latestUpdateInfo": null, + "status": "approved resource", + "resourceExtras": { + "eoscIFGuidelines": null + }, + "id": "riga_stradins_university.rsu_dataverse", + "service": { + "id": "riga_stradins_university.rsu_dataverse", + "abbreviation": "RSU Dataverse", + "name": "RSU Dataverse", + "resourceOrganisation": "riga_stradins_university", + "resourceProviders": [], + "webpage": "https://dataverse.rsu.lv/", + "alternativeIdentifiers": null, + "description": "RSU Dataverse is an institutional repository created based on open source code software provided by Harvard Dataverse. RSU Dataverse is registered in the Registry of Research Data Repositories. RSU Dataverse is created for RSU researchers to deposit their data there after the end of research projects or other research activities, especially in cases when there are no trusted field specific repositories. Data sets deposited in RSU Dataverse can be openly accessible or with restricted or closed access. It is always possible to contact authors of data sets for cooperation and sharing data sets. RSU Dataverse includes three verses - Medicine, Public Health and Social Sciences. It is possible to deposit data up to 50 GB.\n", + "tagline": "RSU Dataverse is created to enable researchers to deposit and publish their data according to FAIR principles, at the same time ensuring security of the data.", + "logo": "https://www.rsu.lv/sites/default/files/imce/rsu_logo_en.png", + "multimedia": [], + "useCases": [], + "scientificDomains": [ + { + "scientificDomain": "scientific_domain-medical_and_health_sciences", + "scientificSubdomain": "scientific_subdomain-medical_and_health_sciences-basic_medicine" + }, + { + "scientificDomain": "scientific_domain-medical_and_health_sciences", + "scientificSubdomain": "scientific_subdomain-medical_and_health_sciences-health_sciences" + }, + { + "scientificDomain": "scientific_domain-medical_and_health_sciences", + "scientificSubdomain": "scientific_subdomain-medical_and_health_sciences-medical_biotechnology" + }, + { + "scientificDomain": "scientific_domain-social_sciences", + "scientificSubdomain": "scientific_subdomain-social_sciences-economics_and_business" + }, + { + "scientificDomain": "scientific_domain-social_sciences", + "scientificSubdomain": "scientific_subdomain-social_sciences-educational_sciences" + }, + { + "scientificDomain": "scientific_domain-social_sciences", + "scientificSubdomain": "scientific_subdomain-social_sciences-law" + }, + { + "scientificDomain": "scientific_domain-social_sciences", + "scientificSubdomain": "scientific_subdomain-social_sciences-media_and_communications" + }, + { + "scientificDomain": "scientific_domain-social_sciences", + "scientificSubdomain": "scientific_subdomain-social_sciences-political_sciences" + }, + { + "scientificDomain": "scientific_domain-social_sciences", + "scientificSubdomain": "scientific_subdomain-social_sciences-psychology" + }, + { + "scientificDomain": "scientific_domain-social_sciences", + "scientificSubdomain": "scientific_subdomain-social_sciences-sociology" + } + ], + "categories": [ + { + "category": "category-access_physical_and_eInfrastructures-data_storage", + "subcategory": "subcategory-access_physical_and_eInfrastructures-data_storage-data" + } + ], + "targetUsers": [ + "target_user-research_communities", + "target_user-research_groups", + "target_user-research_managers", + "target_user-research_networks", + "target_user-research_organisations", + "target_user-research_projects", + "target_user-researchers" + ], + "accessTypes": [ + "access_type-remote" + ], + "accessModes": [ + "access_mode-free_conditionally" + ], + "tags": [ + "e-INFRA non-commercial" + ], + "horizontalService": true, + "serviceCategories": [ + "service_category-storage" + ], + "marketplaceLocations": [ + "marketplace_location-publish_research_outputs", + "marketplace_location-publish_research_outputs" + ], + "geographicalAvailabilities": [ + "WW" + ], + "languageAvailabilities": [ + "en" + ], + "resourceGeographicLocations": [], + "mainContact": { + "firstName": "Ixxxxxx", + "lastName": "Kxxxxxxxxx", + "email": "Ixxxxxx.Kxxxxxxxxx@xxx.xx", + "phone": null, + "position": "Research project coordinator", + "organisation": "riga_stradins_university" + }, + "publicContacts": [ + { + "firstName": null, + "lastName": null, + "email": "dataxxxxx@xxx.xx", + "phone": null, + "position": null, + "organisation": "riga_stradins_university" + } + ], + "helpdeskEmail": "dataxxxxx@xxx.xx", + "securityContactEmail": "dataxxxxx@xxx.xx", + "trl": "trl-9", + "lifeCycleStatus": null, + "certifications": [], + "standards": [], + "openSourceTechnologies": [], + "version": null, + "lastUpdate": null, + "changeLog": [], + "requiredResources": [], + "relatedResources": [], + "relatedPlatforms": [], + "catalogueId": "eosc", + "fundingBody": [], + "fundingPrograms": [], + "grantProjectNames": [], + "helpdeskPage": null, + "userManual": null, + "termsOfUse": "https://wiki.eoscfuture.eu/display/PUBLIC/About+ToU+and+PP", + "privacyPolicy": "https://wiki.eoscfuture.eu/display/PUBLIC/About+ToU+and+PP", + "accessPolicy": null, + "resourceLevel": null, + "trainingInformation": null, + "statusMonitoring": null, + "maintenance": null, + "orderType": "order_type-open_access", + "order": null, + "paymentModel": null, + "pricing": null + } } ] } From f1cb4d9e243a9a5bfd5d76fbd622cdf984ddc3f3 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 16 Nov 2023 21:23:43 +0100 Subject: [PATCH 3/4] fix test assertsfor newly constructed and parsed horizontalServices endpoint --- tests/test_topofeed.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/test_topofeed.py b/tests/test_topofeed.py index 95801d81..b4e82335 100644 --- a/tests/test_topofeed.py +++ b/tests/test_topofeed.py @@ -813,6 +813,15 @@ def test_groupGroups(self): 'provider_tags': 'Open Science' }, 'type': 'PROJECT' + }, + { + 'group': 'riga_stradins_university', + 'subgroup': 'riga_stradins_university.rsu_dataverse', + 'tags': { + 'info_projectname': 'RIga Stradins University', + 'provider_tags': 'university, Latvia, Health, Medicine' + }, + 'type': 'PROJECT' } ]) @@ -903,11 +912,26 @@ def test_groupEndpoints(self): 'info_groupname': 'OpenAIRE Validator', 'service_tags': 'OAI-PMH protocol, horizontalService' }, - 'type': 'SERVICEGROUPS'} + 'type': 'SERVICEGROUPS' + }, + { + 'group': 'riga_stradins_university.rsu_dataverse', + 'hostname': 'dataverse.rsu.lv_riga_stradins_university.rsu_dataverse', + 'service': 'eu.eosc.portal.services.url', + 'tags': { + 'hostname': 'dataverse.rsu.lv', + 'info_ID': 'riga_stradins_university.rsu_dataverse', + 'info_URL': 'https://dataverse.rsu.lv/', + 'info_groupname': 'RSU Dataverse', + 'service_tags': 'e-INFRA non-commercial, horizontalService' + }, + 'type': 'SERVICEGROUPS' + } ]) def test_idGroupname(self): self.assertEqual(self.id_groupname, { + 'riga_stradins_university.rsu_dataverse': 'RSU Dataverse', 'openaire.validator': 'OpenAIRE Validator', 'srce.3dbionotes': '3DBionotes-WS-TEST', 'srce.poem': 'POEM', From 6b8c1fb65c6c3da94456c0bf5a53bd30514fac78 Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Fri, 1 Dec 2023 09:20:32 +0100 Subject: [PATCH 4/4] Version 2.2.4 --- CHANGELOG.md | 6 ++++++ argo-connectors.spec | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5607ac67..8657165b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [2.2.4] - 2023-12-01 + +### Changed + +* ARGO-4428 Parse horizontalService flag now under main service bucket + ## [2.2.3] - 2023-06-13 ### Added diff --git a/argo-connectors.spec b/argo-connectors.spec index 2e87e2dc..e31f2e1e 100644 --- a/argo-connectors.spec +++ b/argo-connectors.spec @@ -2,7 +2,7 @@ %global __python /usr/bin/python3 Name: argo-connectors -Version: 2.2.3 +Version: 2.2.4 Release: 1%{?dist} Group: EGI/SA4 License: ASL 2.0