From 5a60c0d644f36379e4108bc917f56db69791eb40 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 30 Nov 2020 16:01:31 +0100 Subject: [PATCH 01/10] Removed deprecated consume_service method --- nevermined_sdk_py/gateway/gateway.py | 41 ---------------------------- 1 file changed, 41 deletions(-) diff --git a/nevermined_sdk_py/gateway/gateway.py b/nevermined_sdk_py/gateway/gateway.py index 515326d..e4e5d2e 100644 --- a/nevermined_sdk_py/gateway/gateway.py +++ b/nevermined_sdk_py/gateway/gateway.py @@ -82,47 +82,6 @@ def encrypt_files_dict(files_dict, encrypt_endpoint, asset_id, method): return json.loads(response.text)['hash'] - @staticmethod - def consume_service(did, service_agreement_id, service_endpoint, account, files, - destination_folder, index=None): - """ - Call the Gateway endpoint to get access to the different files that form the asset. - - :param service_agreement_id: Service Agreement Id, str - :param service_endpoint: Url to access, str - :param account: Account instance of the consumer signing this agreement, hex-str - :param files: List containing the files to be consumed, list - :param index: Index of the document that is going to be downloaded, int - :param destination_folder: Path, str - :return: True if was downloaded, bool - """ - signature = Keeper.get_instance().sign_hash( - add_ethereum_prefix_and_hash_msg(service_agreement_id), - account) - headers = dict({ - 'X-Consumer-Address': account.address, - 'X-Signature': signature, - 'X-DID': did - }) - - if index is not None: - assert isinstance(index, int), logger.error('index has to be an integer.') - assert index >= 0, logger.error('index has to be 0 or a positive integer.') - assert index < len(files), logger.error( - 'index can not be bigger than the number of files') - consume_url = Gateway._create_access_url(service_endpoint, service_agreement_id, index) - logger.info(f'invoke access endpoint with this url: {consume_url}') - response = Gateway._http_client.get(consume_url, headers=headers, stream=True) - file_name = Gateway._get_file_name(response) - Gateway.write_file(response, destination_folder, file_name) - else: - for i, _file in enumerate(files): - consume_url = Gateway._create_access_url(service_endpoint, service_agreement_id, i) - logger.info(f'invoke access endpoint with this url: {consume_url}') - response = Gateway._http_client.get(consume_url, headers=headers, stream=True) - file_name = Gateway._get_file_name(response) - Gateway.write_file(response, destination_folder, file_name or f'file-{i}') - @staticmethod def access_service(did, service_agreement_id, service_endpoint, account, destination_folder, index): signature = Keeper.get_instance().sign_hash( From 267f368a0bcd09c633bdccf35582e825e4236695 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 30 Nov 2020 18:29:56 +0100 Subject: [PATCH 02/10] Added oauth support to the access endpoint --- nevermined_sdk_py/assets/asset_consumer.py | 5 ++- nevermined_sdk_py/gateway/gateway.py | 48 ++++++++++++++++++---- nevermined_sdk_py/nevermined/assets.py | 1 + 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/nevermined_sdk_py/assets/asset_consumer.py b/nevermined_sdk_py/assets/asset_consumer.py index 230a46d..9f12fa6 100644 --- a/nevermined_sdk_py/assets/asset_consumer.py +++ b/nevermined_sdk_py/assets/asset_consumer.py @@ -13,7 +13,7 @@ class AssetConsumer: @staticmethod def access(service_agreement_id, service_index, ddo, consumer_account, destination, - gateway, secret_store, index=None): + gateway, secret_store, config, index=None): """ Download asset data files or result files from a compute job. @@ -24,6 +24,7 @@ def access(service_agreement_id, service_index, ddo, consumer_account, destinati :param destination: Path, str :param gateway: Gateway instance :param secret_store: SecretStore instance + :param config: Sdk configuration instance :param index: Index of the document that is going to be downloaded, int :return: Asset folder path, str """ @@ -55,6 +56,7 @@ def access(service_agreement_id, service_index, ddo, consumer_account, destinati consume_url, consumer_account, asset_folder, + config, index ) else: @@ -65,6 +67,7 @@ def access(service_agreement_id, service_index, ddo, consumer_account, destinati consume_url, consumer_account, asset_folder, + config, i ) diff --git a/nevermined_sdk_py/gateway/gateway.py b/nevermined_sdk_py/gateway/gateway.py index e4e5d2e..376202b 100644 --- a/nevermined_sdk_py/gateway/gateway.py +++ b/nevermined_sdk_py/gateway/gateway.py @@ -6,6 +6,7 @@ from common_utils_py.agreements.service_agreement import ServiceAgreement from common_utils_py.exceptions import EncryptAssetUrlsError from common_utils_py.http_requests.requests_session import get_requests_session +from common_utils_py.oauth2.token import NeverminedJWTBearerGrant, generate_access_grant_token from contracts_lib_py.utils import add_ethereum_prefix_and_hash_msg from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper @@ -25,6 +26,11 @@ class Gateway: """ _http_client = get_requests_session() + _tokens_cache = {} + + @staticmethod + def _generate_cache_key(*args): + return ''.join(args) @staticmethod def set_http_client(http_client): @@ -83,21 +89,25 @@ def encrypt_files_dict(files_dict, encrypt_endpoint, asset_id, method): return json.loads(response.text)['hash'] @staticmethod - def access_service(did, service_agreement_id, service_endpoint, account, destination_folder, index): - signature = Keeper.get_instance().sign_hash( - add_ethereum_prefix_and_hash_msg(service_agreement_id), - account) - headers = dict({ - 'X-Consumer-Address': account.address, - 'X-Signature': signature, - 'X-DID': did - }) + def access_service(did, service_agreement_id, service_endpoint, account, destination_folder, config, index): + cache_key = Gateway._generate_cache_key(account.address, service_agreement_id, did) + if cache_key not in Gateway._tokens_cache: + grant_token = generate_access_grant_token(account, service_agreement_id, did) + access_token = Gateway.fetch_token(grant_token, config) + Gateway._tokens_cache[cache_key] = access_token + else: + access_token = Gateway._tokens_cache[cache_key] + consume_url = Gateway._create_access_url(service_endpoint, service_agreement_id, index) + headers = {"Authorization": f"Bearer {access_token}"} + response = Gateway._http_client.get(consume_url, headers=headers, stream=True) if response.status_code != 200: raise ValueError(response.text) + file_name = Gateway._get_file_name(response) Gateway.write_file(response, destination_folder, file_name or f'file-{index}') + return response @staticmethod @@ -221,6 +231,16 @@ def execute_compute_service(service_agreement_id, service_endpoint, account, wor raise ValueError(response.text) return response + @staticmethod + def fetch_token(grant_token, config): + fetch_token_url = Gateway.get_fetch_token_endpoint(config) + response = Gateway._http_client.post(fetch_token_url, data={ + "grant_type": NeverminedJWTBearerGrant.GRANT_TYPE, + "assertion": grant_token + }) + if not response.ok: + raise ValueError(response.text) + return response.json()["access_token"] @staticmethod def _prepare_consume_payload(did, service_agreement_id, service_index, signature, @@ -351,6 +371,16 @@ def get_publish_endpoint(config): """ return f'{Gateway.get_gateway_url(config)}/services/publish' + @staticmethod + def get_fetch_token_endpoint(config): + """ + Return the url to fetch an access token. + + :param config: Config + :return: Url, str + """ + return f'{Gateway.get_gateway_url(config)}/services/oauth/token' + @staticmethod def get_rsa_public_key(config): return Gateway._http_client.get(config.gateway_url).json()['rsa-public-key'] diff --git a/nevermined_sdk_py/nevermined/assets.py b/nevermined_sdk_py/nevermined/assets.py index 59b1342..8e381eb 100644 --- a/nevermined_sdk_py/nevermined/assets.py +++ b/nevermined_sdk_py/nevermined/assets.py @@ -420,6 +420,7 @@ def access(self, service_agreement_id, did, service_index, consumer_account, destination, GatewayProvider.get_gateway(), self._get_secret_store(consumer_account), + self._config, index ) From dbb976dce8490c39b314d2e40adef3c6534fa7e9 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Mon, 30 Nov 2020 18:41:41 +0100 Subject: [PATCH 03/10] Added oauth support to download endpoint --- nevermined_sdk_py/gateway/gateway.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/nevermined_sdk_py/gateway/gateway.py b/nevermined_sdk_py/gateway/gateway.py index 376202b..53279ba 100644 --- a/nevermined_sdk_py/gateway/gateway.py +++ b/nevermined_sdk_py/gateway/gateway.py @@ -6,7 +6,7 @@ from common_utils_py.agreements.service_agreement import ServiceAgreement from common_utils_py.exceptions import EncryptAssetUrlsError from common_utils_py.http_requests.requests_session import get_requests_session -from common_utils_py.oauth2.token import NeverminedJWTBearerGrant, generate_access_grant_token +from common_utils_py.oauth2.token import NeverminedJWTBearerGrant, generate_access_grant_token, generate_download_grant_token from contracts_lib_py.utils import add_ethereum_prefix_and_hash_msg from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper @@ -102,7 +102,7 @@ def access_service(did, service_agreement_id, service_endpoint, account, destina headers = {"Authorization": f"Bearer {access_token}"} response = Gateway._http_client.get(consume_url, headers=headers, stream=True) - if response.status_code != 200: + if not response.ok: raise ValueError(response.text) file_name = Gateway._get_file_name(response) @@ -182,19 +182,24 @@ def download(did, account, destination_folder, index, config): :py:class:`requests.Response`: HTTP server response """ - signature = Keeper.get_instance().sign_hash( - add_ethereum_prefix_and_hash_msg(did), account) - headers = { - 'X-Consumer-Address': account.address, - 'X-Signature': signature, - 'X-DID': did - } + cache_key = Gateway._generate_cache_key(account.address, did) + if cache_key not in Gateway._tokens_cache: + grant_token = generate_download_grant_token(account, did) + access_token = Gateway.fetch_token(grant_token, config) + Gateway._tokens_cache[cache_key] = access_token + else: + access_token = Gateway._tokens_cache[cache_key] + + headers = {"Authorization": f"Bearer {access_token}"} consume_url = Gateway._create_download_url(config, index) + response = Gateway._http_client.get(consume_url, headers=headers, stream=True) - if response.status_code != 200: + if not response.ok: raise ValueError(response.text) + file_name = Gateway._get_file_name(response) Gateway.write_file(response, destination_folder, file_name or f'file-{index}') + return response @staticmethod From 54e3b5c114d00e5cfdac22bd2862cab523387ec3 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 2 Dec 2020 09:11:40 +0100 Subject: [PATCH 04/10] Added oauth support for execute endpoint --- nevermined_sdk_py/assets/asset_executor.py | 4 +- nevermined_sdk_py/gateway/gateway.py | 26 +++++++------ nevermined_sdk_py/nevermined/assets.py | 3 +- tests/nevermined/test_assets.py | 45 ++++++++++++++++------ 4 files changed, 51 insertions(+), 27 deletions(-) diff --git a/nevermined_sdk_py/assets/asset_executor.py b/nevermined_sdk_py/assets/asset_executor.py index fa9b814..e6858e1 100644 --- a/nevermined_sdk_py/assets/asset_executor.py +++ b/nevermined_sdk_py/assets/asset_executor.py @@ -10,7 +10,7 @@ class AssetExecutor: """Class representing the call to the Gateway execute endpoint.""" @staticmethod - def execute(agreement_id, compute_ddo, workflow_ddo, consumer_account, gateway, index): + def execute(agreement_id, compute_ddo, workflow_ddo, consumer_account, gateway, index, config): """ :param agreement_id: @@ -22,6 +22,6 @@ def execute(agreement_id, compute_ddo, workflow_ddo, consumer_account, gateway, service_endpoint = ServiceAgreement.from_ddo(ServiceTypes.CLOUD_COMPUTE, compute_ddo).service_endpoint response = gateway.execute_compute_service(agreement_id, service_endpoint, - consumer_account, workflow_ddo) + consumer_account, workflow_ddo, config) return response.json()["workflowId"] diff --git a/nevermined_sdk_py/gateway/gateway.py b/nevermined_sdk_py/gateway/gateway.py index 53279ba..c78f802 100644 --- a/nevermined_sdk_py/gateway/gateway.py +++ b/nevermined_sdk_py/gateway/gateway.py @@ -6,7 +6,7 @@ from common_utils_py.agreements.service_agreement import ServiceAgreement from common_utils_py.exceptions import EncryptAssetUrlsError from common_utils_py.http_requests.requests_session import get_requests_session -from common_utils_py.oauth2.token import NeverminedJWTBearerGrant, generate_access_grant_token, generate_download_grant_token +from common_utils_py.oauth2.token import NeverminedJWTBearerGrant, generate_access_grant_token, generate_download_grant_token, generate_execute_grant_token from contracts_lib_py.utils import add_ethereum_prefix_and_hash_msg from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper @@ -221,18 +221,20 @@ def execute_service(service_agreement_id, service_endpoint, account, workflow_dd return response @staticmethod - def execute_compute_service(service_agreement_id, service_endpoint, account, workflow_ddo): - signature = Keeper.get_instance().sign_hash( - add_ethereum_prefix_and_hash_msg(service_agreement_id), - account) - headers = dict({ - 'X-Consumer-Address': account.address, - 'X-Signature': signature, - 'X-Workflow-DID': workflow_ddo.did - }) + def execute_compute_service(service_agreement_id, service_endpoint, account, workflow_ddo, config): + cache_key = Gateway._generate_cache_key(account.address, service_agreement_id, workflow_ddo.did) + if cache_key not in Gateway._tokens_cache: + grant_token = generate_execute_grant_token(account, service_agreement_id, workflow_ddo.did) + access_token = Gateway.fetch_token(grant_token, config) + Gateway._tokens_cache[cache_key] = access_token + else: + access_token = Gateway._tokens_cache[cache_key] + + headers = {"Authorization": f"Bearer {access_token}"} execute_url = Gateway._create_compute_url(service_endpoint, service_agreement_id) - response = Gateway._http_client.post(execute_url, headers= headers) - if response.status_code != 200: + + response = Gateway._http_client.post(execute_url, headers=headers) + if not response.ok: raise ValueError(response.text) return response diff --git a/nevermined_sdk_py/nevermined/assets.py b/nevermined_sdk_py/nevermined/assets.py index 8e381eb..7ecf0d3 100644 --- a/nevermined_sdk_py/nevermined/assets.py +++ b/nevermined_sdk_py/nevermined/assets.py @@ -602,7 +602,8 @@ def execute(self, agreement_id, did, index, consumer_account, workflow_did): workflow_ddo, consumer_account, GatewayProvider.get_gateway(), - index + index, + self._config ) @staticmethod diff --git a/tests/nevermined/test_assets.py b/tests/nevermined/test_assets.py index 04a8785..23d04bc 100644 --- a/tests/nevermined/test_assets.py +++ b/tests/nevermined/test_assets.py @@ -1,4 +1,5 @@ import logging +from contracts_lib_py.keeper import Keeper import pytest from common_utils_py.agreements.service_agreement import ServiceAgreement @@ -296,22 +297,42 @@ def test_grant_permissions(publisher_instance, metadata, consumer_instance): assert not publisher_instance.assets.get_permissions(ddo.did, consumer.address) -def test_execute_workflow(publisher_instance, consumer_instance): - publisher = publisher_instance.main_account - consumer = consumer_instance.main_account +def test_execute_workflow(publisher_instance_no_init, consumer_instance_no_init): + consumer = publisher_instance_no_init.main_account + publisher = consumer_instance_no_init.main_account + + # publish compute + metadata = get_metadata() + ddo_computing = publisher_instance_no_init.assets.create_compute(metadata, publisher) + + # publish algorithm + metadata = get_algorithm_ddo()['service'][0] + ddo_algorithm = consumer_instance_no_init.assets.create(metadata['attributes'], consumer) + metadata = get_workflow_ddo()['service'][0] - workflow_ddo = publisher_instance.assets.create(metadata['attributes'], publisher) + metadata['attributes']['main']['workflow']['stages'][0]['input'][0]['id'] = ddo_computing.did + metadata['attributes']['main']['workflow']['stages'][0]['transformation']['id'] = ddo_algorithm.did + workflow_ddo = consumer_instance_no_init.assets.create(metadata['attributes'], publisher) assert workflow_ddo - metadata = get_metadata() - ddo_computing = publisher_instance.assets.create_compute(metadata, publisher) - assert ddo_computing + + # order compute asset service = ddo_computing.get_service(service_type=ServiceTypes.CLOUD_COMPUTE) sa = ServiceAgreement.from_service_dict(service.as_dictionary()) - agreement_id = consumer_instance.assets.order(ddo_computing.did, sa.index, consumer) - consumer_instance.assets.execute(agreement_id, ddo_computing.did, sa.index, consumer, - workflow_ddo.did) - publisher_instance.assets.retire(ddo_computing.did) - publisher_instance.assets.retire(workflow_ddo.did) + agreement_id = consumer_instance_no_init.assets.order(ddo_computing.did, sa.index, consumer) + + keeper = Keeper.get_instance() + event = keeper.lock_reward_condition.subscribe_condition_fulfilled( + agreement_id, 60, None, (), wait=True + ) + assert event is not None, "Reward condition is not found" + + # execute workflow + execution_id = consumer_instance_no_init.assets.execute(agreement_id, ddo_computing.did, sa.index, consumer, + workflow_ddo.did) + assert execution_id + + publisher_instance_no_init.assets.retire(ddo_computing.did) + publisher_instance_no_init.assets.retire(workflow_ddo.did) def test_agreement_direct(publisher_instance, consumer_instance, metadata): From 76f4a0608a9d2852c38d5e4521a7521c21a39c6e Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 2 Dec 2020 10:41:04 +0100 Subject: [PATCH 05/10] Added oauth support for compute status endpoint --- nevermined_sdk_py/gateway/gateway.py | 24 ++++++++++------ tests/nevermined/test_assets.py | 41 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/nevermined_sdk_py/gateway/gateway.py b/nevermined_sdk_py/gateway/gateway.py index c78f802..89ab58f 100644 --- a/nevermined_sdk_py/gateway/gateway.py +++ b/nevermined_sdk_py/gateway/gateway.py @@ -6,9 +6,12 @@ from common_utils_py.agreements.service_agreement import ServiceAgreement from common_utils_py.exceptions import EncryptAssetUrlsError from common_utils_py.http_requests.requests_session import get_requests_session -from common_utils_py.oauth2.token import NeverminedJWTBearerGrant, generate_access_grant_token, generate_download_grant_token, generate_execute_grant_token +from common_utils_py.oauth2.token import (NeverminedJWTBearerGrant, + generate_access_grant_token, + generate_download_grant_token, + generate_execute_grant_token, + generate_compute_grant_token) from contracts_lib_py.utils import add_ethereum_prefix_and_hash_msg - from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper logger = logging.getLogger(__name__) @@ -153,14 +156,17 @@ def compute_status(service_agreement_id, execution_id, account, config): :py:class:`requests.Response`: HTTP server response """ - signature = Keeper.get_instance().sign_hash( - add_ethereum_prefix_and_hash_msg(execution_id), - account) - headers = { - 'X-Consumer-Address': account.address, - 'X-Signature': signature, - } + cache_key = Gateway._generate_cache_key(account.address, service_agreement_id, execution_id) + if cache_key not in Gateway._tokens_cache: + grant_token = generate_compute_grant_token(account, service_agreement_id, execution_id) + access_token = Gateway.fetch_token(grant_token, config) + Gateway._tokens_cache[cache_key] = access_token + else: + access_token = Gateway._tokens_cache[cache_key] + + headers = {"Authorization": f"Bearer {access_token}"} consume_url = Gateway._create_compute_status_url(config, service_agreement_id, execution_id) + response = Gateway._http_client.get(consume_url, headers=headers) if response.status_code != 200: raise ValueError(response.text) diff --git a/tests/nevermined/test_assets.py b/tests/nevermined/test_assets.py index 23d04bc..4140a11 100644 --- a/tests/nevermined/test_assets.py +++ b/tests/nevermined/test_assets.py @@ -335,6 +335,47 @@ def test_execute_workflow(publisher_instance_no_init, consumer_instance_no_init) publisher_instance_no_init.assets.retire(workflow_ddo.did) +def test_compute_status(publisher_instance_no_init, consumer_instance_no_init): + consumer = publisher_instance_no_init.main_account + publisher = consumer_instance_no_init.main_account + + # publish compute + metadata = get_metadata() + ddo_computing = publisher_instance_no_init.assets.create_compute(metadata, publisher) + + # publish algorithm + metadata = get_algorithm_ddo()['service'][0] + ddo_algorithm = consumer_instance_no_init.assets.create(metadata['attributes'], consumer) + + metadata = get_workflow_ddo()['service'][0] + metadata['attributes']['main']['workflow']['stages'][0]['input'][0]['id'] = ddo_computing.did + metadata['attributes']['main']['workflow']['stages'][0]['transformation']['id'] = ddo_algorithm.did + workflow_ddo = consumer_instance_no_init.assets.create(metadata['attributes'], publisher) + assert workflow_ddo + + # order compute asset + service = ddo_computing.get_service(service_type=ServiceTypes.CLOUD_COMPUTE) + sa = ServiceAgreement.from_service_dict(service.as_dictionary()) + agreement_id = consumer_instance_no_init.assets.order(ddo_computing.did, sa.index, consumer) + + keeper = Keeper.get_instance() + event = keeper.lock_reward_condition.subscribe_condition_fulfilled( + agreement_id, 60, None, (), wait=True + ) + assert event is not None, "Reward condition is not found" + + # execute workflow + execution_id = consumer_instance_no_init.assets.execute(agreement_id, ddo_computing.did, sa.index, consumer, + workflow_ddo.did) + + # get status + status = consumer_instance_no_init.assets.compute_status(agreement_id, execution_id, consumer) + assert status + + publisher_instance_no_init.assets.retire(ddo_computing.did) + publisher_instance_no_init.assets.retire(workflow_ddo.did) + + def test_agreement_direct(publisher_instance, consumer_instance, metadata): publisher_account = publisher_instance.main_account consumer_account = consumer_instance.main_account From 7174d53b18f758b300bfd73fdb55ebeef5b5190b Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Wed, 2 Dec 2020 14:14:09 +0100 Subject: [PATCH 06/10] Added oauth support to compute logs endpoint --- nevermined_sdk_py/gateway/gateway.py | 19 +++++++------ tests/nevermined/test_assets.py | 42 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/nevermined_sdk_py/gateway/gateway.py b/nevermined_sdk_py/gateway/gateway.py index 89ab58f..e7fb05b 100644 --- a/nevermined_sdk_py/gateway/gateway.py +++ b/nevermined_sdk_py/gateway/gateway.py @@ -128,16 +128,19 @@ def compute_logs(service_agreement_id, execution_id, account, config): :py:class:`requests.Response`: HTTP server response """ - signature = Keeper.get_instance().sign_hash( - add_ethereum_prefix_and_hash_msg(execution_id), - account) - headers = { - 'X-Consumer-Address': account.address, - 'X-Signature': signature, - } + cache_key = Gateway._generate_cache_key(account.address, service_agreement_id, execution_id) + if cache_key not in Gateway._tokens_cache: + grant_token = generate_compute_grant_token(account, service_agreement_id, execution_id) + access_token = Gateway.fetch_token(grant_token, config) + Gateway._tokens_cache[cache_key] = access_token + else: + access_token = Gateway._tokens_cache[cache_key] + + headers = {"Authorization": f"Bearer {access_token}"} consume_url = Gateway._create_compute_logs_url(config, service_agreement_id, execution_id) + response = Gateway._http_client.get(consume_url, headers=headers) - if response.status_code != 200: + if not response.ok: raise ValueError(response.text) return response diff --git a/tests/nevermined/test_assets.py b/tests/nevermined/test_assets.py index 4140a11..ce18b88 100644 --- a/tests/nevermined/test_assets.py +++ b/tests/nevermined/test_assets.py @@ -373,9 +373,51 @@ def test_compute_status(publisher_instance_no_init, consumer_instance_no_init): assert status publisher_instance_no_init.assets.retire(ddo_computing.did) + publisher_instance_no_init.assets.retire(ddo_algorithm.did) publisher_instance_no_init.assets.retire(workflow_ddo.did) +def test_compute_logs(publisher_instance_no_init, consumer_instance_no_init): + consumer = publisher_instance_no_init.main_account + publisher = consumer_instance_no_init.main_account + + # publish compute + metadata = get_metadata() + ddo_computing = publisher_instance_no_init.assets.create_compute(metadata, publisher) + + # publish algorithm + metadata = get_algorithm_ddo()['service'][0] + ddo_algorithm = consumer_instance_no_init.assets.create(metadata['attributes'], consumer) + + metadata = get_workflow_ddo()['service'][0] + metadata['attributes']['main']['workflow']['stages'][0]['input'][0]['id'] = ddo_computing.did + metadata['attributes']['main']['workflow']['stages'][0]['transformation']['id'] = ddo_algorithm.did + workflow_ddo = consumer_instance_no_init.assets.create(metadata['attributes'], publisher) + assert workflow_ddo + + # order compute asset + service = ddo_computing.get_service(service_type=ServiceTypes.CLOUD_COMPUTE) + sa = ServiceAgreement.from_service_dict(service.as_dictionary()) + agreement_id = consumer_instance_no_init.assets.order(ddo_computing.did, sa.index, consumer) + + keeper = Keeper.get_instance() + event = keeper.lock_reward_condition.subscribe_condition_fulfilled( + agreement_id, 60, None, (), wait=True + ) + assert event is not None, "Reward condition is not found" + + # execute workflow + execution_id = consumer_instance_no_init.assets.execute(agreement_id, ddo_computing.did, sa.index, consumer, + workflow_ddo.did) + + # get logs + logs = consumer_instance_no_init.assets.compute_logs(agreement_id, execution_id, consumer) + assert logs + + publisher_instance_no_init.assets.retire(ddo_computing.did) + publisher_instance_no_init.assets.retire(ddo_algorithm.did) + publisher_instance_no_init.assets.retire(workflow_ddo.did) + def test_agreement_direct(publisher_instance, consumer_instance, metadata): publisher_account = publisher_instance.main_account consumer_account = consumer_instance.main_account From bac52176226edbb01e937df11dba6cd61607060f Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 3 Dec 2020 12:14:51 +0100 Subject: [PATCH 07/10] Enabled compute stack in the CI - Fixed tests - Switched consumer and provider accounts in envs --- .github/workflows/pythonpackage.yml | 43 +++++++++++++++++++---------- setup.py | 4 +-- tests/nevermined/test_assets.py | 13 +++++---- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 29b5e57..4d2324a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -23,29 +23,44 @@ jobs: uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + + - name: Install minikube v1.12.0 + run: | + wget https://storage.googleapis.com/minikube/releases/v1.12.0/minikube-linux-amd64 + chmod +x minikube-linux-amd64 + sudo mv minikube-linux-amd64 /usr/local/bin/minikube + + - name: Reclaim some disk space + run : | + docker system prune --all --volumes -f + + - name: Start Nevermined run: | docker login -u ${{ secrets.NEVERMINED_DOCKER_USERNAME }} -p ${{ secrets.NEVERMINED_DOCKER_TOKEN}} git clone https://github.com/nevermined-io/tools nevermined-tools cd nevermined-tools rm -rf "${HOME}/.nevermined/nevermined-contracts/artifacts" - bash -x start_nevermined.sh --no-commons --no-faucet --latest 2>&1 > start_nevermined.log & + # start nevermined with the compute stack + ./start_nevermined.sh --latest --no-commons --local-spree-node --events-handler --compute & + # wait for the compute api to be online. + # the compute api is the last service to come online + ./scripts/wait_for_compute_api.sh + # extract contracts cd .. - for i in $(seq 1 50); do - sleep 5 - [ -f "${HOME}/.nevermined/nvermind-contracts/artifacts/ready" ] && break - done - ls -la "${HOME}/.nevermined/nevermined-contracts/artifacts/" ./scripts/wait_for_migration_and_extract_keeper_artifacts.sh - python -m pip install --upgrade pip + + - name: Install python package + run: | + pip install pip==20.2.4 pip install -r requirements_dev.txt - name: Test with pytest run: | - export PARITY_ADDRESS=0x00bd138abd70e2f00903268f3db08f2d25677c9e - export PARITY_PASSWORD=node0 - export PARITY_KEYFILE=tests/resources/data/key_file_2.json - export PARITY_ADDRESS1=0x068ed00cf0441e4829d9784fcbe7b9e26d4bd8d0 - export PARITY_PASSWORD1=secret - export PARITY_KEYFILE1=tests/resources/data/key_file_1.json + export PARITY_ADDRESS=0x068ed00cf0441e4829d9784fcbe7b9e26d4bd8d0 + export PARITY_PASSWORD=secret + export PARITY_KEYFILE=tests/resources/data/key_file_1.json + export PARITY_ADDRESS1=0x00bd138abd70e2f00903268f3db08f2d25677c9e + export PARITY_PASSWORD1=node0 + export PARITY_KEYFILE1=tests/resources/data/key_file_2.json + pip install pytest pytest -v diff --git a/setup.py b/setup.py index af3628b..6939e71 100644 --- a/setup.py +++ b/setup.py @@ -20,10 +20,10 @@ 'pyopenssl', 'PyJWT', # not jwt 'PyYAML==4.2b4', - 'common-utils-py==0.3.0', + 'common-utils-py==0.4.1', 'contracts-lib-py==0.5.0', 'ocean-secret-store-client==0.0.2', - 'requests==2.21.0', + 'requests~=2.21.0', 'deprecated', 'pycryptodomex', 'tqdm', diff --git a/tests/nevermined/test_assets.py b/tests/nevermined/test_assets.py index ce18b88..26902b2 100644 --- a/tests/nevermined/test_assets.py +++ b/tests/nevermined/test_assets.py @@ -298,8 +298,8 @@ def test_grant_permissions(publisher_instance, metadata, consumer_instance): def test_execute_workflow(publisher_instance_no_init, consumer_instance_no_init): - consumer = publisher_instance_no_init.main_account - publisher = consumer_instance_no_init.main_account + publisher = publisher_instance_no_init.main_account + consumer = consumer_instance_no_init.main_account # publish compute metadata = get_metadata() @@ -332,12 +332,13 @@ def test_execute_workflow(publisher_instance_no_init, consumer_instance_no_init) assert execution_id publisher_instance_no_init.assets.retire(ddo_computing.did) + publisher_instance_no_init.assets.retire(ddo_algorithm.did) publisher_instance_no_init.assets.retire(workflow_ddo.did) def test_compute_status(publisher_instance_no_init, consumer_instance_no_init): - consumer = publisher_instance_no_init.main_account - publisher = consumer_instance_no_init.main_account + publisher = publisher_instance_no_init.main_account + consumer = consumer_instance_no_init.main_account # publish compute metadata = get_metadata() @@ -378,8 +379,8 @@ def test_compute_status(publisher_instance_no_init, consumer_instance_no_init): def test_compute_logs(publisher_instance_no_init, consumer_instance_no_init): - consumer = publisher_instance_no_init.main_account - publisher = consumer_instance_no_init.main_account + publisher = publisher_instance_no_init.main_account + consumer = consumer_instance_no_init.main_account # publish compute metadata = get_metadata() From 3b6895a571111e24f861952b74c389387d5d22b9 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Thu, 3 Dec 2020 17:04:32 +0100 Subject: [PATCH 08/10] Fixed failing tests --- .github/workflows/pythonpackage.yml | 14 ++++---- tests/conftest.py | 46 ++++++++++++++++++++++++-- tests/nevermined/test_agreements.py | 34 +------------------- tests/nevermined/test_assets.py | 50 ++++++++++++++--------------- 4 files changed, 75 insertions(+), 69 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4d2324a..e9e310f 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -55,12 +55,12 @@ jobs: pip install -r requirements_dev.txt - name: Test with pytest run: | - export PARITY_ADDRESS=0x068ed00cf0441e4829d9784fcbe7b9e26d4bd8d0 - export PARITY_PASSWORD=secret - export PARITY_KEYFILE=tests/resources/data/key_file_1.json - export PARITY_ADDRESS1=0x00bd138abd70e2f00903268f3db08f2d25677c9e - export PARITY_PASSWORD1=node0 - export PARITY_KEYFILE1=tests/resources/data/key_file_2.json + export PARITY_ADDRESS=0x00bd138abd70e2f00903268f3db08f2d25677c9e + export PARITY_PASSWORD=node0 + export PARITY_KEYFILE=tests/resources/data/key_file_2.json + export PARITY_ADDRESS1=0x068ed00cf0441e4829d9784fcbe7b9e26d4bd8d0 + export PARITY_PASSWORD1=secret + export PARITY_KEYFILE1=tests/resources/data/key_file_1.json pip install pytest - pytest -v + pytest -v \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 12de013..cc5347a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ import uuid +from unittest.mock import Mock, MagicMock import pytest -from common_utils_py.agreements.service_agreement import ServiceAgreement +from common_utils_py.agreements.service_agreement import ServiceAgreement, ServiceAgreementTemplate from common_utils_py.agreements.service_types import ServiceTypes from common_utils_py.did import DID from common_utils_py.metadata.metadata import Metadata @@ -11,10 +12,13 @@ from examples import ExampleConfig from nevermined_sdk_py import ConfigProvider from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper -from tests.resources.helper_functions import (_get_asset, get_consumer_account, +from nevermined_sdk_py.assets.asset_executor import AssetExecutor +from nevermined_sdk_py.assets.asset_consumer import AssetConsumer +from nevermined_sdk_py.nevermined.agreements import Agreements +from tests.resources.helper_functions import (_get_asset, get_algorithm_ddo, get_consumer_account, get_consumer_instance, get_ddo_sample, get_metadata, get_publisher_account, - get_publisher_instance, get_registered_ddo, + get_publisher_instance, get_registered_ddo, get_workflow_ddo, setup_logging) from tests.resources.mocks.secret_store_mock import SecretStoreMock @@ -109,6 +113,19 @@ def metadata(): return metadata +@pytest.fixture +def algorithm_ddo(): + ddo = get_algorithm_ddo() + ddo['service'][0]['attributes']['main']['checksum'] = str(uuid.uuid4()) + return ddo + + +@pytest.fixture +def workflow_ddo(): + ddo = get_workflow_ddo() + ddo['service'][0]['attributes']['main']['checksum'] = str(uuid.uuid4()) + return ddo + @pytest.fixture def setup_agreements_enviroment(ddo_sample): consumer_acc = get_consumer_account() @@ -145,3 +162,26 @@ def setup_agreements_enviroment(ddo_sample): service_agreement, (lock_cond_id, access_cond_id, escrow_cond_id), ) + + +@pytest.fixture +def agreements(): + publisher_acc = get_publisher_account() + keeper = Keeper.get_instance() + w3 = Web3Provider.get_web3() + did_resolver = Mock() + ddo = get_ddo_sample() + service = ddo.get_service(ServiceTypes.ASSET_ACCESS) + service.update_value( + ServiceAgreementTemplate.TEMPLATE_ID_KEY, + w3.toChecksumAddress(publisher_acc.address) + ) + did_resolver.resolve = MagicMock(return_value=ddo) + + return Agreements( + keeper, + did_resolver, + AssetConsumer, + AssetExecutor, + ConfigProvider.get_config() + ) \ No newline at end of file diff --git a/tests/nevermined/test_agreements.py b/tests/nevermined/test_agreements.py index f36ee17..a4c46aa 100644 --- a/tests/nevermined/test_agreements.py +++ b/tests/nevermined/test_agreements.py @@ -1,44 +1,12 @@ -from unittest.mock import MagicMock, Mock - -import pytest from common_utils_py.agreements.service_agreement import ServiceAgreement -from common_utils_py.agreements.service_agreement_template import ServiceAgreementTemplate from common_utils_py.agreements.service_types import ServiceTypes, ServiceTypesIndices -from contracts_lib_py.web3_provider import Web3Provider -from nevermined_sdk_py import ConfigProvider -from nevermined_sdk_py.assets.asset_consumer import AssetConsumer -from nevermined_sdk_py.assets.asset_executor import AssetExecutor from nevermined_sdk_py.gateway.gateway import Gateway -from nevermined_sdk_py.nevermined.agreements import Agreements from nevermined_sdk_py.nevermined.keeper import NeverminedKeeper as Keeper -from tests.resources.helper_functions import (get_ddo_sample, log_event) +from tests.resources.helper_functions import log_event from tests.resources.mocks.gateway_mock import GatewayMock -@pytest.fixture -def agreements(): - keeper = Keeper.get_instance() - w3 = Web3Provider.get_web3() - did_resolver = Mock() - ddo = get_ddo_sample() - service = ddo.get_service(ServiceTypes.ASSET_ACCESS) - service.update_value( - ServiceAgreementTemplate.TEMPLATE_ID_KEY, - w3.toChecksumAddress("0x00bd138abd70e2f00903268f3db08f2d25677c9e") - ) - did_resolver.resolve = MagicMock(return_value=ddo) - # consumer_class = Mock - # consumer_class.download = MagicMock(return_value='') - return Agreements( - keeper, - did_resolver, - AssetConsumer, - AssetExecutor, - ConfigProvider.get_config() - ) - - def test_sign_agreement(publisher_instance, consumer_instance, registered_ddo): # point consumer_instance's Gateway mock to the publisher's nevermined instance Gateway.set_http_client( diff --git a/tests/nevermined/test_assets.py b/tests/nevermined/test_assets.py index 26902b2..4d394fd 100644 --- a/tests/nevermined/test_assets.py +++ b/tests/nevermined/test_assets.py @@ -9,8 +9,7 @@ from contracts_lib_py.exceptions import DIDNotFound from contracts_lib_py.web3_provider import Web3Provider -from tests.resources.helper_functions import (get_algorithm_ddo, get_workflow_ddo, - log_event, get_metadata) +from tests.resources.helper_functions import log_event def create_asset(publisher_instance, ddo_sample): @@ -249,27 +248,26 @@ def test_assets_validate(publisher_instance, metadata): assert publisher_instance.assets.validate(metadata) -def test_assets_algorithm(publisher_instance): +def test_assets_algorithm(publisher_instance, algorithm_ddo): # Allow publish an algorithm publisher = publisher_instance.main_account - metadata = get_algorithm_ddo()['service'][0] + metadata = algorithm_ddo['service'][0] ddo = publisher_instance.assets.create(metadata['attributes'], publisher) assert ddo publisher_instance.assets.retire(ddo.did) -def test_assets_workflow(publisher_instance): +def test_assets_workflow(publisher_instance, workflow_ddo): # Allow publish an workflow publisher = publisher_instance.main_account - metadata = get_workflow_ddo()['service'][0] + metadata = workflow_ddo['service'][0] ddo = publisher_instance.assets.create(metadata['attributes'], publisher) assert ddo publisher_instance.assets.retire(ddo.did) -def test_assets_compute(publisher_instance): +def test_assets_compute(publisher_instance, metadata): publisher = publisher_instance.main_account - metadata = get_metadata() ddo = publisher_instance.assets.create_compute(metadata, publisher) assert ddo publisher_instance.assets.retire(ddo.did) @@ -290,26 +288,28 @@ def test_grant_permissions(publisher_instance, metadata, consumer_instance): publisher = publisher_instance.main_account consumer = consumer_instance.main_account ddo = publisher_instance.assets.create(metadata, publisher) + assert not publisher_instance.assets.get_permissions(ddo.did, consumer.address) publisher_instance.assets.delegate_persmission(ddo.did, consumer.address, publisher) assert publisher_instance.assets.get_permissions(ddo.did, consumer.address) publisher_instance.assets.revoke_permissions(ddo.did, consumer.address, publisher) assert not publisher_instance.assets.get_permissions(ddo.did, consumer.address) + publisher_instance.assets.retire(ddo.did) + -def test_execute_workflow(publisher_instance_no_init, consumer_instance_no_init): - publisher = publisher_instance_no_init.main_account - consumer = consumer_instance_no_init.main_account +def test_execute_workflow(publisher_instance_no_init, consumer_instance_no_init, metadata, algorithm_ddo, workflow_ddo): + consumer = publisher_instance_no_init.main_account + publisher = consumer_instance_no_init.main_account # publish compute - metadata = get_metadata() ddo_computing = publisher_instance_no_init.assets.create_compute(metadata, publisher) # publish algorithm - metadata = get_algorithm_ddo()['service'][0] + metadata = algorithm_ddo['service'][0] ddo_algorithm = consumer_instance_no_init.assets.create(metadata['attributes'], consumer) - metadata = get_workflow_ddo()['service'][0] + metadata = workflow_ddo['service'][0] metadata['attributes']['main']['workflow']['stages'][0]['input'][0]['id'] = ddo_computing.did metadata['attributes']['main']['workflow']['stages'][0]['transformation']['id'] = ddo_algorithm.did workflow_ddo = consumer_instance_no_init.assets.create(metadata['attributes'], publisher) @@ -336,19 +336,18 @@ def test_execute_workflow(publisher_instance_no_init, consumer_instance_no_init) publisher_instance_no_init.assets.retire(workflow_ddo.did) -def test_compute_status(publisher_instance_no_init, consumer_instance_no_init): - publisher = publisher_instance_no_init.main_account - consumer = consumer_instance_no_init.main_account +def test_compute_status(publisher_instance_no_init, consumer_instance_no_init, metadata, algorithm_ddo, workflow_ddo): + consumer = publisher_instance_no_init.main_account + publisher = consumer_instance_no_init.main_account # publish compute - metadata = get_metadata() ddo_computing = publisher_instance_no_init.assets.create_compute(metadata, publisher) # publish algorithm - metadata = get_algorithm_ddo()['service'][0] + metadata = algorithm_ddo['service'][0] ddo_algorithm = consumer_instance_no_init.assets.create(metadata['attributes'], consumer) - metadata = get_workflow_ddo()['service'][0] + metadata = workflow_ddo['service'][0] metadata['attributes']['main']['workflow']['stages'][0]['input'][0]['id'] = ddo_computing.did metadata['attributes']['main']['workflow']['stages'][0]['transformation']['id'] = ddo_algorithm.did workflow_ddo = consumer_instance_no_init.assets.create(metadata['attributes'], publisher) @@ -378,19 +377,18 @@ def test_compute_status(publisher_instance_no_init, consumer_instance_no_init): publisher_instance_no_init.assets.retire(workflow_ddo.did) -def test_compute_logs(publisher_instance_no_init, consumer_instance_no_init): - publisher = publisher_instance_no_init.main_account - consumer = consumer_instance_no_init.main_account +def test_compute_logs(publisher_instance_no_init, consumer_instance_no_init, metadata, algorithm_ddo, workflow_ddo): + consumer = publisher_instance_no_init.main_account + publisher = consumer_instance_no_init.main_account # publish compute - metadata = get_metadata() ddo_computing = publisher_instance_no_init.assets.create_compute(metadata, publisher) # publish algorithm - metadata = get_algorithm_ddo()['service'][0] + metadata = algorithm_ddo['service'][0] ddo_algorithm = consumer_instance_no_init.assets.create(metadata['attributes'], consumer) - metadata = get_workflow_ddo()['service'][0] + metadata = workflow_ddo['service'][0] metadata['attributes']['main']['workflow']['stages'][0]['input'][0]['id'] = ddo_computing.did metadata['attributes']['main']['workflow']['stages'][0]['transformation']['id'] = ddo_algorithm.did workflow_ddo = consumer_instance_no_init.assets.create(metadata['attributes'], publisher) From 600fe73da2ebc16b6569c6e7deb9bda650a6bd02 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Fri, 4 Dec 2020 09:11:46 +0100 Subject: [PATCH 09/10] Fix contracts-lib version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index da3c22e..48b2231 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ 'PyJWT', # not jwt 'PyYAML==4.2b4', 'common-utils-py==0.4.1', - 'contracts-lib-py==0.5.0', + 'contracts-lib-py==0.5.2', 'nevermined-secret-store==0.1.0', 'requests~=2.21.0', 'deprecated', From 5119ba8e9410ca840a8ee5d60ab39c991b1589b3 Mon Sep 17 00:00:00 2001 From: Rodolphe Marques Date: Fri, 4 Dec 2020 10:51:14 +0100 Subject: [PATCH 10/10] Remove deprecated test --- tests/nevermined/test_assets.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/nevermined/test_assets.py b/tests/nevermined/test_assets.py index 1d6e04a..8d1edc9 100644 --- a/tests/nevermined/test_assets.py +++ b/tests/nevermined/test_assets.py @@ -244,10 +244,6 @@ def test_assets_search(publisher_instance, metadata): publisher_instance.assets.retire(ddo.did) -def test_assets_validate(publisher_instance, metadata): - assert publisher_instance.assets.validate(metadata) - - def test_assets_algorithm(publisher_instance, algorithm_ddo): # Allow publish an algorithm publisher = publisher_instance.main_account