Merge pull request #5214 from ministryofjustice/dsos-2618-create-poc-…

…cross-account-cloudwatch

Dsos 2618 create poc cross account cloudwatch

terraform/environments/hmpps-oem/locals.tf

@@ -217,4 +217,13 @@ locals {
   }
 
   baseline_sns_topics = {}
+
+  environment_cloudwatch_monitoring_options = {
+    development   = local.development_cloudwatch_monitoring_options 
+    test          = local.test_cloudwatch_monitoring_options
+    preproduction = local.preproduction_cloudwatch_monitoring_options
+    production    = local.production_cloudwatch_monitoring_options
+  }
+
+  cloudwatch_monitoring_options = local.environment_cloudwatch_monitoring_options[local.environment]
 }

terraform/environments/hmpps-oem/locals_development.tf

@@ -1,6 +1,11 @@
 # nomis-development environment settings
 locals {
 
+  # cloudwatch monitoring config
+  development_cloudwatch_monitoring_options = {
+    enable_cloudwatch_monitoring_account = false
+  }
+
   # baseline presets config
   development_baseline_presets_options = {
     enable_ec2_delius_dba_secrets_access = true # additional permissions to access delius secrets

terraform/environments/hmpps-oem/locals_preproduction.tf

@@ -1,6 +1,11 @@
 # nomis-preproduction environment settings
 locals {
 
+  # cloudwatch monitoring config
+  preproduction_cloudwatch_monitoring_options = {
+    enable_cloudwatch_monitoring_account = false
+  }
+
   # baseline presets config
   preproduction_baseline_presets_options = {}
 

terraform/environments/hmpps-oem/locals_production.tf

@@ -1,6 +1,11 @@
 # nomis-production environment settings
 locals {
 
+  # cloudwatch monitoring config
+  production_cloudwatch_monitoring_options = {
+    enable_cloudwatch_monitoring_account = false
+  }
+
   # baseline presets config
   production_baseline_presets_options = {}
 

terraform/environments/hmpps-oem/locals_test.tf

@@ -1,6 +1,11 @@
 # nomis-test environment settings
 locals {
 
+  # cloudwatch monitoring config
+  test_cloudwatch_monitoring_options = {
+    enable_cloudwatch_monitoring_account = true
+  }
+
   # baseline presets config
   test_baseline_presets_options = {}
 

terraform/environments/hmpps-oem/main.tf

@@ -162,3 +162,8 @@ module "baseline" {
     lookup(local.baseline_environment_config, "baseline_ssm_parameters", {}),
   )
 }
+
+module "cloudwatch_dashboard" {
+  source  = "./modules/cloudwatch_dashboard"
+  options = local.cloudwatch_monitoring_options
+}

terraform/environments/hmpps-oem/modules/cloudwatch_dashboard/dashboard.tf

@@ -0,0 +1,274 @@
+resource "aws_cloudwatch_dashboard" "hmpps-oem_cloudwatch_dashboard" {
+  dashboard_name = "CloudWatch-Default"
+  dashboard_body = jsonencode(local.dashboard_body)
+}
+
+locals {
+
+  cloudwatch_period = 300
+  region            = "eu-west-2"
+
+  dashboard_body = {
+    widgets = [
+      local.EC2CPUUtil,
+      local.EC2MemoryUtil,
+      local.EC2DiskUsed,
+      local.LoadBalancerTargetResponseTime,
+      local.LoadBalancerRequestCount,
+      local.LoadBalancerHTTP5XXsCount,
+      local.EBSVolumeDiskIOPS,
+      local.EBSVolumeDiskThroughput,
+      local.AllEBSVolumeStats,
+      local.LBGraphedMetricsHeading,
+      local.EC2GraphedMetricsHeading,
+      local.EBSGraphedMetricsHeading,
+    ]
+  }
+
+  EC2CPUUtil = {
+    type   = "metric"
+    x      = 0
+    y      = 1
+    width  = 7
+    height = 8
+    properties = {
+      view    = "timeSeries"
+      stacked = false
+      region  = local.region
+      title   = "Top 5 instances by highest CPU Utilization %"
+      stat    = "Maximum"
+      period  = local.cloudwatch_period
+      metrics = [
+        [{ "expression" : "SELECT MAX(CPUUtilization)\nFROM SCHEMA(\"AWS/EC2\", InstanceId)\nGROUP BY InstanceId\nORDER BY MAX() DESC\nLIMIT 5", "label" : "", "id" : "q1" }]
+      ]
+    }
+  }
+
+  EC2MemoryUtil = {
+    type   = "metric"
+    x      = 7
+    y      = 1
+    width  = 6
+    height = 8
+    properties = {
+      view    = "bar"
+      stacked = false
+      region  = local.region
+      title   = "EC2 Memory Utilization %"
+      stat    = "Maximum"
+      period  = local.cloudwatch_period
+      metrics = [
+        [{ "expression" : "SELECT MAX(mem_used_percent) FROM SCHEMA(CWAgent, InstanceId,name,server_type) GROUP BY InstanceId ORDER BY MAX() DESC", "label" : "", "id" : "q1", "yAxis" : "left" }]
+      ]
+    }
+  }
+
+  EC2DiskUsed = {
+    type   = "metric"
+    x      = 13
+    y      = 1
+    width  = 6
+    height = 8
+    properties = {
+      view    = "timeSeries"
+      stacked = false
+      region  = local.region
+      title   = "EC2 Disk Used %"
+      stat    = "Maximum"
+      period  = local.cloudwatch_period
+      metrics = [
+        [{ "expression" : "SELECT MAX(disk_used_percent) FROM SCHEMA(CWAgent, InstanceId) GROUP BY InstanceId ORDER BY MAX() DESC", "label" : "", "id" : "q1" }]
+      ]
+    }
+  }
+
+  LoadBalancerTargetResponseTime = {
+    type   = "metric"
+    x      = 0
+    y      = 10
+    width  = 7
+    height = 8
+    properties = {
+      view    = "timeSeries"
+      stacked = true
+      region  = local.region
+      title   = "LoadBalancer Target Response Time"
+      stat    = "Maximum"
+      period  = local.cloudwatch_period
+      metrics = [
+        [{ "expression" : "SELECT MAX(TargetResponseTime) FROM SCHEMA(\"AWS/ApplicationELB\", LoadBalancer,TargetGroup) GROUP BY TargetGroup ORDER BY MAX() DESC", "label" : "", "id" : "q1" }]
+      ]
+    }
+  }
+
+  LoadBalancerRequestCount = {
+    type   = "metric"
+    x      = 7
+    y      = 10
+    width  = 6
+    height = 8
+    properties = {
+      view    = "timeSeries"
+      stacked = true
+      region  = local.region
+      title   = "LoadBalancer Request Count"
+      stat    = "Maximum"
+      period  = local.cloudwatch_period
+      metrics = [
+        [{ "expression" : "SELECT COUNT(RequestCount) FROM \"AWS/ApplicationELB\" GROUP BY LoadBalancer ORDER BY COUNT() DESC", "label" : "", "id" : "q1" }]
+      ]
+    }
+  }
+
+  LoadBalancerHTTP5XXsCount = {
+    type   = "metric"
+    x      = 13
+    y      = 10
+    width  = 6
+    height = 8
+    properties = {
+      view    = "timeSeries"
+      stacked = true
+      region  = local.region
+      title   = "LoadBalancer HTTP 5XXs Count"
+      stat    = "Maximum"
+      period  = local.cloudwatch_period
+      metrics = [
+        [{ "expression" : "SELECT COUNT(HTTPCode_ELB_5XX_Count) FROM SCHEMA(\"AWS/ApplicationELB\", AvailabilityZone,LoadBalancer,TargetGroup) GROUP BY LoadBalancer ORDER BY COUNT() DESC", "label" : "", "id" : "q1" }]
+      ]
+    }
+  }
+
+  EBSVolumeDiskIOPS = {
+    type   = "metric"
+    x      = 0
+    y      = 19
+    width  = 12
+    height = 6
+    properties = {
+      view    = "timeSeries"
+      stacked = false
+      region  = local.region
+      title   = "EBS Volumes Total IOPs"
+      stat    = "Sum"
+      period  = local.cloudwatch_period
+      metrics = [
+        [{ "expression" : "m1/PERIOD(m1)", "label" : "Read IOPs", "id" : "e1" }],
+        [{ "expression" : "m2/PERIOD(m2)", "label" : "Write IOPs", "id" : "e2" }],
+        [{ "expression" : "e1+e2", "label" : "Total IOPs", "id" : "e3" }],
+        ["AWS/EBS", "VolumeReadOps", "VolumeId", "*", { "id" : "m1", "visible" : false }],
+        ["AWS/EBS", "VolumeWriteOps", "VolumeId", "*", { "id" : "m2", "visible" : false }]
+      ]
+    }
+  }
+
+  EBSVolumeDiskThroughput = {
+    type   = "metric"
+    x      = 0
+    y      = 25
+    width  = 12
+    height = 6
+    properties = {
+      view    = "timeSeries"
+      stacked = false
+      region  = local.region
+      title   = "EBS Volumes Throughput"
+      stat    = "Sum"
+      period  = local.cloudwatch_period
+      metrics = [
+        [{ "expression" : "SELECT SUM(VolumeWriteBytes)\nFROM SCHEMA(\"AWS/EBS\", VolumeId)\nGROUP BY VolumeId\nORDER BY SUM() DESC\nLIMIT 10", "label" : "VolumeWriteBytes", "id" : "m3", "stat" : "Sum", "visible" : false }],
+        [{ "expression" : "SELECT SUM(VolumeReadBytes) FROM SCHEMA(\"AWS/EBS\", VolumeId) GROUP BY VolumeId ORDER BY SUM() DESC LIMIT 10", "label" : "VolumeReadBytes", "id" : "m4", "stat" : "Sum", "visible" : false }],
+        [{ "expression" : "(m4/(1024*1024))/PERIOD(m4)", "label" : "MB Read Per Second", "id" : "e4" }],
+        [{ "expression" : "(m3/(1024*1024))/PERIOD(m3)", "label" : "MB Write Per Second", "id" : "e5" }],
+        [{ "expression" : "e4+e5", "label" : "Total Consumed MB/s", "id" : "e6" }]
+      ]
+    }
+  }
+
+  AllEBSVolumeStats = {
+    type   = "explorer"
+    x      = 0
+    y      = 31
+    width  = 24
+    height = 15
+    properties = {
+      region = local.region
+      title  = "All EBS Volume Stats"
+      stat   = "Sum"
+      period = local.cloudwatch_period
+      widgetOptions = {
+        view          = "timeSeries"
+        stacked       = false
+        rowsPerPage   = 50
+        widgetsPerRow = 2
+      }
+      labels = [
+        { key : "application", value : "hmpps-oem" }
+      ]
+      metrics = [
+        {
+          "metricName" : "VolumeReadBytes",
+          "resourceType" : "AWS::EC2::Volume",
+          "stat" : "Sum"
+        },
+        {
+          "metricName" : "VolumeWriteBytes",
+          "resourceType" : "AWS::EC2::Volume",
+          "stat" : "Sum"
+        },
+        {
+          "metricName" : "VolumeIdleTime",
+          "resourceType" : "AWS::EC2::Volume",
+          "stat" : "Average"
+        },
+        {
+          "metricName" : "VolumeReadOps",
+          "resourceType" : "AWS::EC2::Volume",
+          "stat" : "Sum"
+        },
+        {
+          "metricName" : "VolumeWriteOps",
+          "resourceType" : "AWS::EC2::Volume",
+          "stat" : "Sum"
+        }
+      ]
+    }
+  }
+
+  LBGraphedMetricsHeading = {
+    type   = "text"
+    x      = 0
+    y      = 9
+    width  = 24
+    height = 1
+    properties = {
+      markdown   = "## LoadBalancer Graphed Metrics"
+      background = "solid"
+    }
+  }
+
+  EC2GraphedMetricsHeading = {
+    type   = "text"
+    x      = 0
+    y      = 0
+    width  = 24
+    height = 1
+    properties = {
+      markdown   = "## EC2 Graphed Metrics"
+      background = "solid"
+    }
+  }
+
+  EBSGraphedMetricsHeading = {
+    type   = "text"
+    x      = 0
+    y      = 18
+    width  = 24
+    height = 1
+    properties = {
+      markdown   = "## EBS Volume Graphed Metrics"
+      background = "solid"
+    }
+  }
+
+}

terraform/environments/hmpps-oem/modules/cloudwatch_dashboard/oam.tf

@@ -0,0 +1,28 @@
+resource "aws_oam_sink" "monitoring_account_oam_sink" {
+  count = var.options.enable_cloudwatch_monitoring_account ? 1 : 0
+  name = "HMPPSOemSink"
+}
+
+resource "aws_oam_sink_policy" "monitoring_account_oam_sink_policy" {
+  count = var.options.enable_cloudwatch_monitoring_account ? 1 : 0
+  sink_identifier = aws_oam_sink.monitoring_account_oam_sink[0].id
+
+  policy = jsonencode ({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = ["oam:CreateLink", "oam:UpdateLink"]
+        Effect = "Allow"
+        Resource = "*"
+        Principal = {
+          "AWS"  = var.source_account_ids
+        }
+        Condition = {
+          "ForAllValues:StringEquals" = {
+            "oam:ResourceTypes" = ["AWS::CloudWatch::Metric"]
+          }
+        }
+      }
+    ]
+  })
+}

terraform/environments/hmpps-oem/modules/cloudwatch_dashboard/variables.tf

@@ -0,0 +1,11 @@
+variable "source_account_ids" {
+  type = list(string)
+  default = ["612659970365", "546088120047"] #nomis-test and oasys-test
+}
+
+variable "options" {
+  description = "Map of options controlling what resources to return"
+  type = object({
+    enable_cloudwatch_monitoring_account = optional(bool, false)
+  })
+}