From 46f0868b8c77bffc0de91a0e6825928811c15bd7 Mon Sep 17 00:00:00 2001 From: Matt <38562764+LavMatt@users.noreply.github.com> Date: Thu, 9 Nov 2023 15:28:52 +0000 Subject: [PATCH] Create metadata add push to catalogue (#2272) * add call to push-to-catalgoue lambda * update tests * upversion Dockerfile, config, changelog * change env name * upversion base image * add structlog to test reqs * better reqs --- containers/daap-create-metadata/CHANGELOG.md | 11 +++++- containers/daap-create-metadata/Dockerfile | 5 ++- containers/daap-create-metadata/config.json | 2 +- .../src/var/task/create_metadata.py | 39 ++++++++++++++++++- .../src/var/task/requirements.txt | 2 + .../tests/requirements.txt | 2 +- .../tests/unit/create_metadata_test.py | 14 +++++-- 7 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 containers/daap-create-metadata/src/var/task/requirements.txt diff --git a/containers/daap-create-metadata/CHANGELOG.md b/containers/daap-create-metadata/CHANGELOG.md index ec258ad8b9..05320abc9e 100644 --- a/containers/daap-create-metadata/CHANGELOG.md +++ b/containers/daap-create-metadata/CHANGELOG.md @@ -9,7 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- update Dockerfile COPY command fixing `LAMBDA_TASK_ROOT` typo +## [2.1.0] 2023-11-06 + +### Added + +- function `push_to_catalogue` which calls another lambda which + pushes registered data product metadata to the catalogue, as + of now openmetadata. +- returns a reposnse body for success (previously this was empty) + and additonal key/values in that reposnse for the catalogue + response. ## [2.0.0] 2023-10-13 diff --git a/containers/daap-create-metadata/Dockerfile b/containers/daap-create-metadata/Dockerfile index 7ad7f9116a..ad18e606ba 100644 --- a/containers/daap-create-metadata/Dockerfile +++ b/containers/daap-create-metadata/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ministryofjustice/data-platform-daap-python-base:4.0.0 +FROM ghcr.io/ministryofjustice/data-platform-daap-python-base:6.1.0 ARG VERSION @@ -6,4 +6,7 @@ ENV VERSION="${VERSION}" COPY src/var/task ${LAMBDA_TASK_ROOT} +RUN python -m pip install --no-cache-dir --upgrade pip==23.3.1 \ + && python -m pip install --no-cache-dir --requirement requirements.txt + CMD ["create_metadata.handler"] diff --git a/containers/daap-create-metadata/config.json b/containers/daap-create-metadata/config.json index cb0d3a1924..36ee70dee3 100644 --- a/containers/daap-create-metadata/config.json +++ b/containers/daap-create-metadata/config.json @@ -1,6 +1,6 @@ { "name": "daap-create-metadata", - "version": "2.0.0", + "version": "2.1.0", "registry": "ecr", "ecr": { "role": "arn:aws:iam::013433889002:role/modernisation-platform-oidc-cicd", diff --git a/containers/daap-create-metadata/src/var/task/create_metadata.py b/containers/daap-create-metadata/src/var/task/create_metadata.py index 45d3be3435..153607575b 100644 --- a/containers/daap-create-metadata/src/var/task/create_metadata.py +++ b/containers/daap-create-metadata/src/var/task/create_metadata.py @@ -2,11 +2,36 @@ import os from http import HTTPMethod, HTTPStatus +import boto3 from data_platform_logging import DataPlatformLogger from data_platform_paths import DataProductConfig from data_product_metadata import DataProductMetadata +def push_to_catalogue( + metadata: dict, + version: str, + data_product_name: str, + table_name: str | None = None, +): + lambda_client = boto3.client("lambda") + + catalogue_input = { + "metadata": metadata, + "version": version, + "data_product_name": data_product_name, + "table_name": table_name, + } + + catalogue_response = lambda_client.invoke( + FunctionName=os.getenv("PUSH_TO_CATALOGUE_LAMBDA_ARN"), + InvocationType="RequestResponse", + Payload=json.dumps(catalogue_input), + ) + + return catalogue_response + + def handler(event, context): """ Handles requests that are passed through the Amazon API Gateway data-platform/register REST API endpoint. @@ -106,7 +131,9 @@ def format_error_response( write_key = DataProductConfig(data_product_name).metadata_path("v1.0").key data_product_metadata.write_json_to_s3(write_key) response_code = HTTPStatus.CREATED - response_body = None + response_body = { + "message": f"{data_product_name} registered in the data platform" + } else: response_code = HTTPStatus.BAD_REQUEST error_message = f"Metadata failed validation with error: {data_product_metadata.error_traceback}" # noqa E501 @@ -116,8 +143,16 @@ def format_error_response( error_message = f"Data Product {data_product_name} already has a version 1 registered metadata." # noqa E501 return format_error_response(response_code, event, error_message) + catalogue_response = push_to_catalogue( + metadata=data_product_metadata.data, + version="v1.0", + data_product_name=data_product_name, + ) + response = format_response( - response_code=response_code, event=event, body_dict=response_body + response_code=response_code, + event=event, + body_dict={**response_body, **catalogue_response}, ) return response diff --git a/containers/daap-create-metadata/src/var/task/requirements.txt b/containers/daap-create-metadata/src/var/task/requirements.txt new file mode 100644 index 0000000000..5045a2e46a --- /dev/null +++ b/containers/daap-create-metadata/src/var/task/requirements.txt @@ -0,0 +1,2 @@ +boto3==1.28.40 +structlog==23.2.0 diff --git a/containers/daap-create-metadata/tests/requirements.txt b/containers/daap-create-metadata/tests/requirements.txt index c91dc96b07..1412d0bf0d 100644 --- a/containers/daap-create-metadata/tests/requirements.txt +++ b/containers/daap-create-metadata/tests/requirements.txt @@ -1,4 +1,4 @@ -boto3==1.28.40 +-r ../src/var/task/requirements.txt botocore==1.31.40 pytest==7.4.1 moto==4.2.2 diff --git a/containers/daap-create-metadata/tests/unit/create_metadata_test.py b/containers/daap-create-metadata/tests/unit/create_metadata_test.py index 66c9e1173b..3945519d08 100644 --- a/containers/daap-create-metadata/tests/unit/create_metadata_test.py +++ b/containers/daap-create-metadata/tests/unit/create_metadata_test.py @@ -32,9 +32,17 @@ def test_metadata_creation_pass(fake_event, fake_context): mock_metadata.return_value.valid = True with patch("create_metadata.DataProductConfig") as mock_key: mock_key.return_value.metadata_path.key = "somekey" - response = create_metadata.handler(event=fake_event, context=fake_context) - - assert response["statusCode"] == HTTPStatus.CREATED + with patch("create_metadata.push_to_catalogue") as mock_push: + mock_push.return_value = {"catalogue_message": "success"} + response = create_metadata.handler( + event=fake_event, context=fake_context + ) + + assert response["statusCode"] == HTTPStatus.CREATED + assert json.loads(response["body"]) == { + "message": "test_name registered in the data platform", + "catalogue_message": "success", + } def test_metadata_validation_fail(fake_event, fake_context):