Merge pull request #279 from ARGOeu/devel

Verion 2.2.4
ARGOeu · Dec 1, 2023 · c496b7f · c496b7f
2 parents 2773f8d + 5759afc
commit c496b7f
Show file tree

Hide file tree

Showing 8 changed files with 302 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## [2.2.4] - 2023-12-01
+
+### Changed
+
+* ARGO-4428 Parse horizontalService flag now under main service bucket
+
 ## [2.2.3] - 2023-06-13
 
 ### Added

diff --git a/argo-connectors.spec b/argo-connectors.spec
@@ -2,7 +2,7 @@
 %global __python /usr/bin/python3
 
 Name:    argo-connectors
-Version: 2.2.3
+Version: 2.2.4
 Release: 1%{?dist}
 Group:   EGI/SA4
 License: ASL 2.0

diff --git a/modules/parse/base.py b/modules/parse/base.py
@@ -122,6 +122,7 @@ def csv_to_json(self, data):
                 continue
             num_item = 0
             datum = {}
+
             for item in header:
                 datum[item] = row[num_item]
                 num_item = num_item + 1

diff --git a/modules/parse/provider_topology.py b/modules/parse/provider_topology.py
@@ -22,7 +22,7 @@ def build_urlpath_id(http_endpoint):
         return uuid.uuid3(uuid.NAMESPACE_URL, path)
     else:
         return None
-    
+
 
 class ParseResources(ParseHelpers):
     def __init__(self, logger, data=None, keys=[], custname=None):
@@ -48,6 +48,10 @@ def _parse_data(self):
                         key_true = extras.get(key, False)
                         if key_true:
                             tags.append(key)
+                for key in self._keys:
+                    key_true = resource.get(key, False)
+                    if key_true:
+                        tags.append(key)
                 if not resource.get('name', False):
                     continue
                 self._resources.append({

diff --git a/modules/tasks/provider_topology.py b/modules/tasks/provider_topology.py
@@ -42,7 +42,7 @@ def _parse(self):
         total = doc['total']
         from_index = doc['from']
         to_index = doc['to']
- 
+
         return total, from_index, to_index
 
 
@@ -88,7 +88,7 @@ def parse_source_extensions(self, extensions, groupnames):
         resources_extended = ParseExtensions(self.logger, extensions, groupnames, self.uidservendp, self.logger.customer)
 
         return resources_extended.get_extensions()
-    
+
     def parse_source_topo(self, resources, providers):
         topo = ParseTopo(self.logger, providers, resources, self.uidservendp, self.logger.customer)
 
@@ -167,7 +167,7 @@ async def fetch_data(self, feed, access_token, paginated):
                                                                             remote_topo.path,
                                                                             from_index,
                                                                             num),
-                                                                            headers=headers)           
+                                                                            headers=headers)
                 await session.close()
                 return res
 

diff --git a/tests/sample-private-provider.json b/tests/sample-private-provider.json
@@ -248,6 +248,101 @@
         ]
       },
       "id": "openaire"
+    },
+    {
+      "metadata": {
+        "registeredBy": "Sxxxxx Uxxxx",
+        "registeredAt": "1626436442264",
+        "modifiedBy": "Axxxxxxx Gxxxxxxxxx",
+        "modifiedAt": "1655453425739",
+        "terms": [
+          "sxxxxx.uxxxx@xxx.xx",
+          "axxxxxxx.gxxxxxxxxx@xxxxxxxx.xx"
+        ],
+        "published": false
+      },
+      "active": true,
+      "suspended": false,
+      "identifiers": null,
+      "migrationStatus": null,
+      "loggingInfo": null,
+      "latestAuditInfo": null,
+      "latestOnboardingInfo": null,
+      "latestUpdateInfo": null,
+      "status": "approved provider",
+      "templateStatus": "approved template",
+      "provider": {
+        "id": "riga_stradins_university",
+        "abbreviation": "RIga Stradins University",
+        "name": "Riga Stradins University",
+        "website": "https://www.rsu.lv/",
+        "legalEntity": true,
+        "legalStatus": "provider_legal_status-public_legal_entity",
+        "hostingLegalEntity": null,
+        "alternativeIdentifiers": null,
+        "description": "Riga Stradins University from Riga, Latvia is one of the leading and most advanced higher education teaching institutions in the Baltic region. The RSU research and teaching staff are professionals in their areas of expertise and are opinion leaders in the society. RSUs mission is to train highly qualified experts in the field of healthcare and social sciences, so they can serve the society of Latvia, the European Union, as well as the world. ",
+        "logo": "https://www.rsu.lv/sites/default/files/imce/rsu_alternate_logo_en.png",
+        "multimedia": [],
+        "scientificDomains": [
+          {
+            "scientificDomain": "scientific_domain-medical_and_health_sciences",
+            "scientificSubdomain": "scientific_subdomain-medical_and_health_sciences-health_sciences"
+          }
+        ],
+        "tags": [
+          "university",
+          "Latvia",
+          "Health",
+          "Medicine"
+        ],
+        "structureTypes": [],
+        "location": {
+          "streetNameAndNumber": "16 Dzirciema Street",
+          "postalCode": " LV-1007",
+          "city": "Riga",
+          "region": "Riga",
+          "country": "LV"
+        },
+        "mainContact": {
+          "firstName": "Ixxxxxx",
+          "lastName": "Kxxxxxxxxx",
+          "email": "Ixxxxxx.Kxxxxxxxxx@rsu.lv",
+          "phone": "+37167455586",
+          "position": "Science project coordinator"
+        },
+        "publicContacts": [
+          {
+            "firstName": "Ixxxxxx",
+            "lastName": "Kxxxxxxxxx",
+            "email": "Ixxxxxx.Kxxxxxxxxx@xxx.xx",
+            "phone": "+xxxxxxxxxxx",
+            "position": "Science project coordinator"
+          }
+        ],
+        "lifeCycleStatus": "provider_life_cycle_status-operational",
+        "certifications": [],
+        "participatingCountries": [
+          "LV"
+        ],
+        "affiliations": [],
+        "networks": [],
+        "catalogueId": "eosc",
+        "esfriDomains": [],
+        "esfriType": null,
+        "merilScientificDomains": [],
+        "areasOfActivity": [],
+        "societalGrandChallenges": [],
+        "nationalRoadmaps": [],
+        "users": [
+          {
+            "id": "",
+            "email": "Ixxxxxx.Kxxxxxxxxx@xxx.xx",
+            "name": "Ixxxxxx",
+            "surname": "Kxxxxxxxxx"
+          }
+        ]
+      },
+      "id": "riga_stradins_university"
     }
   ]
 }
diff --git a/tests/sample-private-resource.json b/tests/sample-private-resource.json
@@ -650,6 +650,172 @@
             "pricing": null
         },
         "id": "openaire.validator"
+    },
+    {
+      "metadata": {
+        "registeredBy": "Ixxxxxx Kxxxxxxxxx",
+        "registeredAt": "1652777979850",
+        "modifiedBy": "Ixxxxxx Kxxxxxxxxx",
+        "modifiedAt": "1652943494220",
+        "terms": null,
+        "published": false
+      },
+      "active": true,
+      "suspended": false,
+      "identifiers": null,
+      "migrationStatus": null,
+      "loggingInfo": null,
+      "latestAuditInfo": null,
+      "latestOnboardingInfo": null,
+      "latestUpdateInfo": null,
+      "status": "approved resource",
+      "resourceExtras": {
+        "eoscIFGuidelines": null
+      },
+      "id": "riga_stradins_university.rsu_dataverse",
+      "service": {
+        "id": "riga_stradins_university.rsu_dataverse",
+        "abbreviation": "RSU Dataverse",
+        "name": "RSU Dataverse",
+        "resourceOrganisation": "riga_stradins_university",
+        "resourceProviders": [],
+        "webpage": "https://dataverse.rsu.lv/",
+        "alternativeIdentifiers": null,
+        "description": "RSU Dataverse is an institutional repository created based on open source code software provided by Harvard Dataverse. RSU Dataverse is registered in the Registry of Research Data Repositories. RSU Dataverse is created for RSU researchers to deposit their data there after the end of research projects or other research activities, especially in cases when there are no trusted field specific repositories. Data sets deposited in RSU Dataverse can be openly accessible or with restricted or closed access. It is always possible to contact authors of data sets for cooperation and sharing data sets. RSU Dataverse includes three verses - Medicine, Public Health and Social Sciences. It is possible to deposit data up to 50 GB.\n",
+        "tagline": "RSU Dataverse is created to enable researchers to deposit and publish their data according to FAIR principles, at the same time ensuring security of the data.",
+        "logo": "https://www.rsu.lv/sites/default/files/imce/rsu_logo_en.png",
+        "multimedia": [],
+        "useCases": [],
+        "scientificDomains": [
+          {
+            "scientificDomain": "scientific_domain-medical_and_health_sciences",
+            "scientificSubdomain": "scientific_subdomain-medical_and_health_sciences-basic_medicine"
+          },
+          {
+            "scientificDomain": "scientific_domain-medical_and_health_sciences",
+            "scientificSubdomain": "scientific_subdomain-medical_and_health_sciences-health_sciences"
+          },
+          {
+            "scientificDomain": "scientific_domain-medical_and_health_sciences",
+            "scientificSubdomain": "scientific_subdomain-medical_and_health_sciences-medical_biotechnology"
+          },
+          {
+            "scientificDomain": "scientific_domain-social_sciences",
+            "scientificSubdomain": "scientific_subdomain-social_sciences-economics_and_business"
+          },
+          {
+            "scientificDomain": "scientific_domain-social_sciences",
+            "scientificSubdomain": "scientific_subdomain-social_sciences-educational_sciences"
+          },
+          {
+            "scientificDomain": "scientific_domain-social_sciences",
+            "scientificSubdomain": "scientific_subdomain-social_sciences-law"
+          },
+          {
+            "scientificDomain": "scientific_domain-social_sciences",
+            "scientificSubdomain": "scientific_subdomain-social_sciences-media_and_communications"
+          },
+          {
+            "scientificDomain": "scientific_domain-social_sciences",
+            "scientificSubdomain": "scientific_subdomain-social_sciences-political_sciences"
+          },
+          {
+            "scientificDomain": "scientific_domain-social_sciences",
+            "scientificSubdomain": "scientific_subdomain-social_sciences-psychology"
+          },
+          {
+            "scientificDomain": "scientific_domain-social_sciences",
+            "scientificSubdomain": "scientific_subdomain-social_sciences-sociology"
+          }
+        ],
+        "categories": [
+          {
+            "category": "category-access_physical_and_eInfrastructures-data_storage",
+            "subcategory": "subcategory-access_physical_and_eInfrastructures-data_storage-data"
+          }
+        ],
+        "targetUsers": [
+          "target_user-research_communities",
+          "target_user-research_groups",
+          "target_user-research_managers",
+          "target_user-research_networks",
+          "target_user-research_organisations",
+          "target_user-research_projects",
+          "target_user-researchers"
+        ],
+        "accessTypes": [
+          "access_type-remote"
+        ],
+        "accessModes": [
+          "access_mode-free_conditionally"
+        ],
+        "tags": [
+          "e-INFRA non-commercial"
+        ],
+        "horizontalService": true,
+        "serviceCategories": [
+          "service_category-storage"
+        ],
+        "marketplaceLocations": [
+          "marketplace_location-publish_research_outputs",
+          "marketplace_location-publish_research_outputs"
+        ],
+        "geographicalAvailabilities": [
+          "WW"
+        ],
+        "languageAvailabilities": [
+          "en"
+        ],
+        "resourceGeographicLocations": [],
+        "mainContact": {
+          "firstName": "Ixxxxxx",
+          "lastName": "Kxxxxxxxxx",
+          "email": "Ixxxxxx.Kxxxxxxxxx@xxx.xx",
+          "phone": null,
+          "position": "Research project coordinator",
+          "organisation": "riga_stradins_university"
+        },
+        "publicContacts": [
+          {
+            "firstName": null,
+            "lastName": null,
+            "email": "dataxxxxx@xxx.xx",
+            "phone": null,
+            "position": null,
+            "organisation": "riga_stradins_university"
+          }
+        ],
+        "helpdeskEmail": "dataxxxxx@xxx.xx",
+        "securityContactEmail": "dataxxxxx@xxx.xx",
+        "trl": "trl-9",
+        "lifeCycleStatus": null,
+        "certifications": [],
+        "standards": [],
+        "openSourceTechnologies": [],
+        "version": null,
+        "lastUpdate": null,
+        "changeLog": [],
+        "requiredResources": [],
+        "relatedResources": [],
+        "relatedPlatforms": [],
+        "catalogueId": "eosc",
+        "fundingBody": [],
+        "fundingPrograms": [],
+        "grantProjectNames": [],
+        "helpdeskPage": null,
+        "userManual": null,
+        "termsOfUse": "https://wiki.eoscfuture.eu/display/PUBLIC/About+ToU+and+PP",
+        "privacyPolicy": "https://wiki.eoscfuture.eu/display/PUBLIC/About+ToU+and+PP",
+        "accessPolicy": null,
+        "resourceLevel": null,
+        "trainingInformation": null,
+        "statusMonitoring": null,
+        "maintenance": null,
+        "orderType": "order_type-open_access",
+        "order": null,
+        "paymentModel": null,
+        "pricing": null
+      }
     }
   ]
 }
diff --git a/tests/test_topofeed.py b/tests/test_topofeed.py
@@ -813,6 +813,15 @@ def test_groupGroups(self):
                     'provider_tags': 'Open Science'
                 },
                 'type': 'PROJECT'
+            },
+            {
+                'group': 'riga_stradins_university',
+                'subgroup': 'riga_stradins_university.rsu_dataverse',
+                'tags': {
+                    'info_projectname': 'RIga Stradins University',
+                    'provider_tags': 'university, Latvia, Health, Medicine'
+                },
+                'type': 'PROJECT'
             }
         ])
 
@@ -903,11 +912,26 @@ def test_groupEndpoints(self):
                     'info_groupname': 'OpenAIRE Validator',
                     'service_tags': 'OAI-PMH protocol, horizontalService'
                 },
-                'type': 'SERVICEGROUPS'}
+                'type': 'SERVICEGROUPS'
+            },
+            {
+                'group': 'riga_stradins_university.rsu_dataverse',
+                'hostname': 'dataverse.rsu.lv_riga_stradins_university.rsu_dataverse',
+                'service': 'eu.eosc.portal.services.url',
+                'tags': {
+                    'hostname': 'dataverse.rsu.lv',
+                    'info_ID': 'riga_stradins_university.rsu_dataverse',
+                    'info_URL': 'https://dataverse.rsu.lv/',
+                    'info_groupname': 'RSU Dataverse',
+                    'service_tags': 'e-INFRA non-commercial, horizontalService'
+                },
+                'type': 'SERVICEGROUPS'
+            }
         ])
 
     def test_idGroupname(self):
         self.assertEqual(self.id_groupname, {
+            'riga_stradins_university.rsu_dataverse': 'RSU Dataverse',
             'openaire.validator': 'OpenAIRE Validator',
             'srce.3dbionotes': '3DBionotes-WS-TEST',
             'srce.poem': 'POEM',