Create metadata add push to catalogue (#2272)

* add call to push-to-catalgoue lambda * update tests * upversion Dockerfile, config, changelog * change env name * upversion base image * add structlog to test reqs * better reqs
ministryofjustice · Nov 9, 2023 · 46f0868 · 46f0868
1 parent 75a73d8
commit 46f0868
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 9 deletions.
diff --git a/containers/daap-create-metadata/CHANGELOG.md b/containers/daap-create-metadata/CHANGELOG.md
@@ -9,7 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-- update Dockerfile COPY command fixing `LAMBDA_TASK_ROOT` typo
+## [2.1.0] 2023-11-06
+
+### Added
+
+- function `push_to_catalogue` which calls another lambda which
+  pushes registered data product metadata to the catalogue, as
+  of now openmetadata.
+- returns a reposnse body for success (previously this was empty)
+  and additonal key/values in that reposnse for the catalogue
+  response.
 
 ## [2.0.0] 2023-10-13
 

diff --git a/containers/daap-create-metadata/Dockerfile b/containers/daap-create-metadata/Dockerfile
@@ -1,9 +1,12 @@
-FROM ghcr.io/ministryofjustice/data-platform-daap-python-base:4.0.0
+FROM ghcr.io/ministryofjustice/data-platform-daap-python-base:6.1.0
 
 ARG VERSION
 
 ENV VERSION="${VERSION}"
 
 COPY src/var/task ${LAMBDA_TASK_ROOT}
 
+RUN python -m pip install --no-cache-dir --upgrade pip==23.3.1 \
+    && python -m pip install --no-cache-dir --requirement requirements.txt
+
 CMD ["create_metadata.handler"]
diff --git a/containers/daap-create-metadata/config.json b/containers/daap-create-metadata/config.json
@@ -1,6 +1,6 @@
 {
   "name": "daap-create-metadata",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "registry": "ecr",
   "ecr": {
     "role": "arn:aws:iam::013433889002:role/modernisation-platform-oidc-cicd",

diff --git a/containers/daap-create-metadata/src/var/task/create_metadata.py b/containers/daap-create-metadata/src/var/task/create_metadata.py
@@ -2,11 +2,36 @@
 import os
 from http import HTTPMethod, HTTPStatus
 
+import boto3
 from data_platform_logging import DataPlatformLogger
 from data_platform_paths import DataProductConfig
 from data_product_metadata import DataProductMetadata
 
 
+def push_to_catalogue(
+    metadata: dict,
+    version: str,
+    data_product_name: str,
+    table_name: str | None = None,
+):
+    lambda_client = boto3.client("lambda")
+
+    catalogue_input = {
+        "metadata": metadata,
+        "version": version,
+        "data_product_name": data_product_name,
+        "table_name": table_name,
+    }
+
+    catalogue_response = lambda_client.invoke(
+        FunctionName=os.getenv("PUSH_TO_CATALOGUE_LAMBDA_ARN"),
+        InvocationType="RequestResponse",
+        Payload=json.dumps(catalogue_input),
+    )
+
+    return catalogue_response
+
+
 def handler(event, context):
     """
     Handles requests that are passed through the Amazon API Gateway data-platform/register REST API endpoint.
@@ -106,7 +131,9 @@ def format_error_response(
             write_key = DataProductConfig(data_product_name).metadata_path("v1.0").key
             data_product_metadata.write_json_to_s3(write_key)
             response_code = HTTPStatus.CREATED
-            response_body = None
+            response_body = {
+                "message": f"{data_product_name} registered in the data platform"
+            }
         else:
             response_code = HTTPStatus.BAD_REQUEST
             error_message = f"Metadata failed validation with error: {data_product_metadata.error_traceback}"  # noqa E501
@@ -116,8 +143,16 @@ def format_error_response(
         error_message = f"Data Product {data_product_name} already has a version 1 registered metadata."  # noqa E501
         return format_error_response(response_code, event, error_message)
 
+    catalogue_response = push_to_catalogue(
+        metadata=data_product_metadata.data,
+        version="v1.0",
+        data_product_name=data_product_name,
+    )
+
     response = format_response(
-        response_code=response_code, event=event, body_dict=response_body
+        response_code=response_code,
+        event=event,
+        body_dict={**response_body, **catalogue_response},
     )
 
     return response
diff --git a/containers/daap-create-metadata/src/var/task/requirements.txt b/containers/daap-create-metadata/src/var/task/requirements.txt
@@ -0,0 +1,2 @@
+boto3==1.28.40
+structlog==23.2.0
diff --git a/containers/daap-create-metadata/tests/requirements.txt b/containers/daap-create-metadata/tests/requirements.txt
@@ -1,4 +1,4 @@
-boto3==1.28.40
+-r ../src/var/task/requirements.txt
 botocore==1.31.40
 pytest==7.4.1
 moto==4.2.2

diff --git a/containers/daap-create-metadata/tests/unit/create_metadata_test.py b/containers/daap-create-metadata/tests/unit/create_metadata_test.py
@@ -32,9 +32,17 @@ def test_metadata_creation_pass(fake_event, fake_context):
         mock_metadata.return_value.valid = True
         with patch("create_metadata.DataProductConfig") as mock_key:
             mock_key.return_value.metadata_path.key = "somekey"
-            response = create_metadata.handler(event=fake_event, context=fake_context)
-
-            assert response["statusCode"] == HTTPStatus.CREATED
+            with patch("create_metadata.push_to_catalogue") as mock_push:
+                mock_push.return_value = {"catalogue_message": "success"}
+                response = create_metadata.handler(
+                    event=fake_event, context=fake_context
+                )
+
+                assert response["statusCode"] == HTTPStatus.CREATED
+                assert json.loads(response["body"]) == {
+                    "message": "test_name registered in the data platform",
+                    "catalogue_message": "success",
+                }
 
 
 def test_metadata_validation_fail(fake_event, fake_context):