diff --git a/bin/existing-eks-mixed-observability.ts b/bin/existing-eks-mixed-observability.ts new file mode 100644 index 00000000..5880e36a --- /dev/null +++ b/bin/existing-eks-mixed-observability.ts @@ -0,0 +1,8 @@ +import ExistingEksMixedConstruct from '../lib/existing-eks-mixed-observability-construct'; +import { configureApp, errorHandler } from '../lib/common/construct-utils'; + +const app = configureApp(); + +new ExistingEksMixedConstruct().buildAsync(app, 'existing-eks-mixed').catch((error) => { + errorHandler(app, "Existing Cluster Pattern is missing information of existing cluster: " + error); +}); \ No newline at end of file diff --git a/docs/patterns/existing-eks-observability-accelerators/existing-eks-mixed-observability.md b/docs/patterns/existing-eks-observability-accelerators/existing-eks-mixed-observability.md new file mode 100644 index 00000000..e756adf4 --- /dev/null +++ b/docs/patterns/existing-eks-observability-accelerators/existing-eks-mixed-observability.md @@ -0,0 +1,78 @@ +# Existing EKS Cluster AWS Mixed Observability Accelerator + +## Architecture + +The following figure illustrates the architecture of the pattern we will be deploying for Existing EKS Cluster AWS Mixed Observability pattern, using AWS native tools such as CloudWatch and X-Ray and Open Source tools such as Amazon Distro for OpenTelmetry (ADOT) and Prometheus Node Exporter. + +![Architecture](../images/mixed-diagram.png) + +This example makes use of CloudWatch, as a metric and log aggregation layer, while X-Ray is used as a trace-aggregation layer. In order to collect the metrics and traces, we use the Open Source ADOT collector. Fluent Bit is used to export the logs to CloudWatch Logs. + +In this architecture, AWS X-Ray provides a complete view of requests as they travel through your application and filters visual data across payloads, functions, traces, services, and APIs. X-Ray also allows you to perform analytics, to gain powerful insights about your distributed trace data. + +Utilizing CloudWatch and X-Ray as an aggregation layer allows for a fully-managed scalable telemetry backend. In this example we get those benefits while still having the flexibility and rapid development of the Open Source collection tools. + +## Objective + +This pattern aims to add Observability on top of an existing EKS cluster, with a mixture of AWS native and open source managed AWS services. + +## Prerequisites: + +Ensure that you have installed the following tools on your machine: + +1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +2. [kubectl](https://Kubernetes.io/docs/tasks/tools/) +3. [cdk](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_install) +4. [npm](https://docs.npmjs.com/cli/v8/commands/npm-install) + +You will also need: + +1. Either an existing EKS cluster, or you can setup a new one with [Single New EKS Cluster Observability Accelerator](../single-new-eks-observability-accelerators/single-new-eks-cluster.md) +2. An OpenID Connect (OIDC) provider, associated to the above EKS cluster (Note: Single EKS Cluster Pattern takes care of that for you) + +## Deploying + +1. Edit `~/.cdk.json` by setting the name of your existing cluster: + +```json + "context": { + ... + "existing.cluster.name": "...", + ... + } +``` + +2. Edit `~/.cdk.json` by setting the kubectl role name; if you used Single New EKS Cluster Observability Accelerator to setup your cluster, the kubectl role name would be provided by the output of the deployment, on your command-line interface (CLI): + +```json + "context": { + ... + "existing.kubectl.rolename":"...", + ... + } +``` + +3. Run the following command from the root of this repository to deploy the pipeline stack: + +```bash +make build +make pattern existing-eks-mixed-observability deploy +``` + +## Verify the resources + +Please see [Single New EKS Cluster AWS Mixed Observability Accelerator](../single-new-eks-observability-accelerators/single-new-eks-mixed-observability.md). + +## Teardown + +You can teardown the whole CDK stack with the following command: + +```bash +make pattern existing-eks-mixed-observability destroy +``` + +If you setup your cluster with Single New EKS Cluster Observability Accelerator, you also need to run: + +```bash +make pattern single-new-eks-cluster destroy +``` diff --git a/lib/existing-eks-mixed-observability-construct/index.ts b/lib/existing-eks-mixed-observability-construct/index.ts new file mode 100644 index 00000000..3fce4577 --- /dev/null +++ b/lib/existing-eks-mixed-observability-construct/index.ts @@ -0,0 +1,63 @@ +import { ImportClusterProvider, utils } from '@aws-quickstart/eks-blueprints'; +import * as blueprints from '@aws-quickstart/eks-blueprints'; +import { cloudWatchDeploymentMode } from '@aws-quickstart/eks-blueprints'; +import { ObservabilityBuilder } from '../common/observability-builder'; +import * as cdk from "aws-cdk-lib"; +import * as eks from 'aws-cdk-lib/aws-eks'; + +export default class ExistingEksMixedobservabilityConstruct { + async buildAsync(scope: cdk.App, id: string) { + // AddOns for the cluster + const stackId = `${id}-observability-accelerator`; + + const clusterName = utils.valueFromContext(scope, "existing.cluster.name", undefined); + const kubectlRoleName = utils.valueFromContext(scope, "existing.kubectl.rolename", undefined); + + const account = process.env.COA_ACCOUNT_ID! || process.env.CDK_DEFAULT_ACCOUNT!; + const region = process.env.COA_AWS_REGION! || process.env.CDK_DEFAULT_REGION!; + + const sdkCluster = await blueprints.describeCluster(clusterName, region); // get cluster information using EKS APIs + const vpcId = sdkCluster.resourcesVpcConfig?.vpcId; + + /** + * Assumes the supplied role is registered in the target cluster for kubectl access. + */ + + const importClusterProvider = new ImportClusterProvider({ + clusterName: sdkCluster.name!, + version: eks.KubernetesVersion.of(sdkCluster.version!), + clusterEndpoint: sdkCluster.endpoint, + openIdConnectProvider: blueprints.getResource(context => + new blueprints.LookupOpenIdConnectProvider(sdkCluster.identity!.oidc!.issuer!).provide(context)), + clusterCertificateAuthorityData: sdkCluster.certificateAuthority?.data, + kubectlRoleArn: blueprints.getResource(context => new blueprints.LookupRoleProvider(kubectlRoleName).provide(context)).roleArn, + clusterSecurityGroupId: sdkCluster.resourcesVpcConfig?.clusterSecurityGroupId + }); + + const cloudWatchAdotAddOn = new blueprints.addons.CloudWatchAdotAddOn({ + deploymentMode: cloudWatchDeploymentMode.DEPLOYMENT, + namespace: 'default', + name: 'adot-collector-cloudwatch', + metricsNameSelectors: ['apiserver_request_.*', 'container_memory_.*', 'container_threads', 'otelcol_process_.*'], + }); + + const addOns: Array = [ + new blueprints.addons.CloudWatchLogsAddon({ + logGroupPrefix: `/aws/eks/${stackId}`, + logRetentionDays: 30 + }), + new blueprints.addons.AdotCollectorAddOn(), + cloudWatchAdotAddOn, + new blueprints.addons.XrayAdotAddOn(), + ]; + + ObservabilityBuilder.builder() + .account(account) + .region(region) + .addExistingClusterObservabilityBuilderAddOns() + .clusterProvider(importClusterProvider) + .resourceProvider(blueprints.GlobalResources.Vpc, new blueprints.VpcProvider(vpcId)) // this is required with import cluster provider + .addOns(...addOns) + .build(scope, stackId); + } +}