diff --git a/terraform/environments/electronic-monitoring-data/ap_airflow_iam.tf b/terraform/environments/electronic-monitoring-data/ap_airflow_iam.tf index 72e15036243..7932563c35c 100644 --- a/terraform/environments/electronic-monitoring-data/ap_airflow_iam.tf +++ b/terraform/environments/electronic-monitoring-data/ap_airflow_iam.tf @@ -22,6 +22,9 @@ module "load_alcohol_monitoring_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "alcohol-monitoring" environment = local.environment database_name = "capita-alcohol-monitoring" @@ -37,6 +40,9 @@ module "load_orca_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "orca" environment = local.environment database_name = "civica-orca" @@ -52,6 +58,9 @@ module "load_atrium_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "atrium" environment = local.environment database_name = "g4s-atrium" @@ -67,6 +76,9 @@ module "load_atv_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "atv" environment = local.environment database_name = "g4s-atv" @@ -82,6 +94,9 @@ module "load_cap_dw_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "cap-dw" environment = local.environment database_name = "g4s-cap-dw" @@ -98,6 +113,9 @@ module "load_emsys_mvp_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "emsys-mvp" environment = local.environment database_name = "g4s-emsys-mvp" @@ -114,6 +132,9 @@ module "load_fep_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "fep" environment = local.environment database_name = "g4s-fep" @@ -129,6 +150,9 @@ module "load_rf_hours_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "rf-hours" environment = local.environment database_name = "g4s-rf-hours" @@ -144,6 +168,9 @@ module "load_subject_history_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "subject-history" environment = local.environment database_name = "g4s-subject-history" @@ -159,6 +186,9 @@ module "load_tasking_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "tasking" environment = local.environment database_name = "g4s-tasking" @@ -174,6 +204,9 @@ module "load_telephony_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "telephony" environment = local.environment database_name = "g4s-telephony" @@ -189,7 +222,10 @@ module "load_unstructured_atrium_database" { count = local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" - name = "unstructured-atrium-database" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + + name = "unstructured-atrium" environment = local.environment database_name = "g4s-atrium-unstructured" path_to_data = "/load/g4s_atrium_unstructured/structure" @@ -205,6 +241,9 @@ module "load_fms" { count = local.is-test || local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "fms" environment = local.environment database_name = "serco-fms" @@ -221,6 +260,9 @@ module "load_mdss" { count = local.is-test || local.is-production ? 1 : 0 source = "./modules/ap_airflow_load_data_iam_role" + data_bucket_lf_resource = aws_lakeformation_resource.data_bucket.arn + de_role_arn = try(one(data.aws_iam_roles.data_engineering_roles.arns)) + name = "mdss" environment = local.environment database_name = "allied-mdss" @@ -230,4 +272,4 @@ module "load_mdss" { oidc_arn = aws_iam_openid_connect_provider.analytical_platform_compute.arn athena_dump_bucket = module.s3-athena-bucket.bucket cadt_bucket = module.s3-create-a-derived-table-bucket.bucket -} \ No newline at end of file +} diff --git a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/main.tf b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/main.tf index ebf0aefebe9..651f65b4a52 100644 --- a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/main.tf @@ -38,7 +38,7 @@ data "aws_iam_policy_document" "oidc_assume_role_policy" { # ----------------------------- resource "aws_iam_role" "role_ap_airflow" { - name = local.role_name + name_prefix = local.role_name description = var.role_description assume_role_policy = data.aws_iam_policy_document.oidc_assume_role_policy.json force_detach_policies = true diff --git a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/outputs.tf b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/outputs.tf new file mode 100644 index 00000000000..d30de0f3f61 --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/outputs.tf @@ -0,0 +1,3 @@ +output "iam_role" { + value = aws_iam_role.role_ap_airflow +} \ No newline at end of file diff --git a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/versions.tf b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/versions.tf new file mode 100644 index 00000000000..2b58ee107cb --- /dev/null +++ b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_iam_role/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + required_version = ">= 1.0.1" +} diff --git a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/main.tf b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/main.tf index d6cc1b6fea8..3723a54382b 100644 --- a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/main.tf +++ b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/main.tf @@ -1,6 +1,7 @@ locals { camel-sid = join("", [for word in split("-", var.name) : title(word)]) - snake-database = replace(var.database_name, "-", "_") + suffix = var.environment == "test" ? "_test" : "" + snake-database = "${replace(var.database_name, "-", "_")}${local.suffix}" } data "aws_region" "current" {} @@ -73,6 +74,12 @@ data "aws_iam_policy_document" "load_data" { "arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:table/${local.snake-database}*/*" ] } + statement { + sid = "GetDataAccessForLakeFormation${local.camel-sid}" + effect = "Allow" + actions = ["lakeformation:GetDataAccess"] + resources = ["*"] + } statement { sid = "ListAccountAlias${local.camel-sid}" effect = "Allow" @@ -87,7 +94,7 @@ data "aws_iam_policy_document" "load_data" { } } -module "load_unstructured_atrium_database" { +module "ap_database_sharing" { source = "../ap_airflow_iam_role" environment = var.environment @@ -98,3 +105,11 @@ module "load_unstructured_atrium_database" { oidc_arn = var.oidc_arn max_session_duration = var.max_session_duration } + +module "share_dbs_with_roles" { + source = "../lakeformation_database_share" + dbs_to_grant = toset([local.snake-database]) + data_bucket_lf_resource = var.data_bucket_lf_resource + role_arn = module.ap_database_sharing.iam_role.arn + de_role_arn = var.de_role_arn +} diff --git a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/variables.tf b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/variables.tf index 1e3fedb4918..5bfe8133f64 100644 --- a/terraform/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/variables.tf +++ b/terraform/environments/electronic-monitoring-data/modules/ap_airflow_load_data_iam_role/variables.tf @@ -50,3 +50,15 @@ variable "max_session_duration" { nullable = true default = 7200 } + +variable "de_role_arn" { + nullable = false + type = string + description = "The arn of the data engineering module" +} + +variable "data_bucket_lf_resource" { + nullable = false + type = string + description = "The arn of the LakeFormation resource where our parquet files are held" +} \ No newline at end of file