Merge pull request #9346 from ministryofjustice/lw/add-lake-formation…

…-permissions-to-ap-airflow-module added the complained about error from yesterday's airflow logs
ministryofjustice · Jan 16, 2025 · 6fbed6a · 6fbed6a
2 parents ce2f291 + ea47cbd
commit 6fbed6a
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 5 deletions.
diff --git a/terraform/environments/electronic-monitoring-data/ap_airflow_iam.tf b/terraform/environments/electronic-monitoring-data/ap_airflow_iam.tf
@@ -22,6 +22,9 @@ module "load_alcohol_monitoring_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "alcohol-monitoring"
   environment        = local.environment
   database_name      = "capita-alcohol-monitoring"
@@ -37,6 +40,9 @@ module "load_orca_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "orca"
   environment        = local.environment
   database_name      = "civica-orca"
@@ -52,6 +58,9 @@ module "load_atrium_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "atrium"
   environment        = local.environment
   database_name      = "g4s-atrium"
@@ -67,6 +76,9 @@ module "load_atv_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "atv"
   environment        = local.environment
   database_name      = "g4s-atv"
@@ -82,6 +94,9 @@ module "load_cap_dw_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name                 = "cap-dw"
   environment          = local.environment
   database_name        = "g4s-cap-dw"
@@ -98,6 +113,9 @@ module "load_emsys_mvp_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name                 = "emsys-mvp"
   environment          = local.environment
   database_name        = "g4s-emsys-mvp"
@@ -114,6 +132,9 @@ module "load_fep_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "fep"
   environment        = local.environment
   database_name      = "g4s-fep"
@@ -129,6 +150,9 @@ module "load_rf_hours_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "rf-hours"
   environment        = local.environment
   database_name      = "g4s-rf-hours"
@@ -144,6 +168,9 @@ module "load_subject_history_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "subject-history"
   environment        = local.environment
   database_name      = "g4s-subject-history"
@@ -159,6 +186,9 @@ module "load_tasking_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "tasking"
   environment        = local.environment
   database_name      = "g4s-tasking"
@@ -174,6 +204,9 @@ module "load_telephony_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "telephony"
   environment        = local.environment
   database_name      = "g4s-telephony"
@@ -189,7 +222,10 @@ module "load_unstructured_atrium_database" {
   count  = local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
-  name               = "unstructured-atrium-database"
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
+  name               = "unstructured-atrium"
   environment        = local.environment
   database_name      = "g4s-atrium-unstructured"
   path_to_data       = "/load/g4s_atrium_unstructured/structure"
@@ -205,6 +241,9 @@ module "load_fms" {
   count  = local.is-test || local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "fms"
   environment        = local.environment
   database_name      = "serco-fms"
@@ -221,6 +260,9 @@ module "load_mdss" {
   count  = local.is-test || local.is-production ? 1 : 0
   source = "./modules/ap_airflow_load_data_iam_role"
 
+  data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn
+  de_role_arn             = try(one(data.aws_iam_roles.data_engineering_roles.arns))
+
   name               = "mdss"
   environment        = local.environment
   database_name      = "allied-mdss"
@@ -230,4 +272,4 @@ module "load_mdss" {
   oidc_arn           = aws_iam_openid_connect_provider.analytical_platform_compute.arn
   athena_dump_bucket = module.s3-athena-bucket.bucket
   cadt_bucket        = module.s3-create-a-derived-table-bucket.bucket
-}
+}
diff --git a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/main.tf b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/main.tf
@@ -38,7 +38,7 @@ data "aws_iam_policy_document" "oidc_assume_role_policy" {
 # -----------------------------
 
 resource "aws_iam_role" "role_ap_airflow" {
-  name                  = local.role_name
+  name_prefix           = local.role_name
   description           = var.role_description
   assume_role_policy    = data.aws_iam_policy_document.oidc_assume_role_policy.json
   force_detach_policies = true

diff --git a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/outputs.tf b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/outputs.tf
@@ -0,0 +1,3 @@
+output "iam_role" {
+  value = aws_iam_role.role_ap_airflow
+}
diff --git a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/versions.tf b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/versions.tf
@@ -0,0 +1,13 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.0"
+    }
+  }
+  required_version = ">= 1.0.1"
+}
diff --git a/...orm/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/main.tf b/...orm/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/main.tf
@@ -1,6 +1,7 @@
 locals {
   camel-sid      = join("", [for word in split("-", var.name) : title(word)])
-  snake-database = replace(var.database_name, "-", "_")
+  suffix         = var.environment == "test" ? "_test" : ""
+  snake-database = "${replace(var.database_name, "-", "_")}${local.suffix}"
 }
 
 data "aws_region" "current" {}
@@ -73,6 +74,12 @@ data "aws_iam_policy_document" "load_data" {
       "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:table/${local.snake-database}*/*"
     ]
   }
+  statement {
+    sid       = "GetDataAccessForLakeFormation${local.camel-sid}"
+    effect    = "Allow"
+    actions   = ["lakeformation:GetDataAccess"]
+    resources = ["*"]
+  }
   statement {
     sid       = "ListAccountAlias${local.camel-sid}"
     effect    = "Allow"
@@ -87,7 +94,7 @@ data "aws_iam_policy_document" "load_data" {
   }
 }
 
-module "load_unstructured_atrium_database" {
+module "ap_database_sharing" {
   source = "../ap_airflow_iam_role"
 
   environment          = var.environment
@@ -98,3 +105,11 @@ module "load_unstructured_atrium_database" {
   oidc_arn             = var.oidc_arn
   max_session_duration = var.max_session_duration
 }
+
+module "share_dbs_with_roles" {
+  source                  = "../lakeformation_database_share"
+  dbs_to_grant            = toset([local.snake-database])
+  data_bucket_lf_resource = var.data_bucket_lf_resource
+  role_arn                = module.ap_database_sharing.iam_role.arn
+  de_role_arn             = var.de_role_arn
+}
diff --git a/...nvironments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/variables.tf b/...nvironments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/variables.tf
@@ -50,3 +50,15 @@ variable "max_session_duration" {
   nullable    = true
   default     = 7200
 }
+
+variable "de_role_arn" {
+  nullable    = false
+  type        = string
+  description = "The arn of the data engineering module"
+}
+
+variable "data_bucket_lf_resource" {
+  nullable    = false
+  type        = string
+  description = "The arn of the LakeFormation resource where our parquet files are held"
+}