cloudformation.yml

AWSTemplateFormatVersion: 2010-09-09

Parameters:
  BucketName:
    Description: Bucket where files will be stored
    Type: String
    Default: otel-files-service

Mappings:
  # AMI: Amazon Linux 2 AMI 64-bit (x86) by region
  RegionMap:
    ap-south-1:
      AMI: ami-019cd93943ccead1a
    eu-north-1:
      AMI: ami-09d3c9501075e4112
    eu-west-3:
      AMI: ami-0584a5ab8f4034679
    eu-west-2:
      AMI: ami-0f9e888df95272f70
    eu-west-1:
      AMI: ami-0fc56b47fc1f238ee
    ap-northeast-3:
      AMI: ami-0217f1c25d44bec66
    ap-northeast-2:
      AMI: ami-07a2318163330ee84
    ap-northeast-1:
      AMI: ami-0dbca050974482176
    ca-central-1:
      AMI: ami-01a9d286de9a4d56e
    sa-east-1:
      AMI: ami-06b07a4baf3024a9f
    ap-southeast-1:
      AMI: ami-047d5a3391704b8b2
    ap-southeast-2:
      AMI: ami-0c372b59cfa8c3d65
    eu-central-1:
      AMI: ami-03b7db59d53c5e228
    us-east-1:
      AMI: ami-0ac664bd64e1dcc6b
    us-east-2:
      AMI: ami-05175b461d18d94d9
    us-west-1:
      AMI: ami-0853d0de3297e47e0
    us-west-2:
      AMI: ami-06c7fbd87fa7b507c


Resources:
  FrontendInstance:
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !FindInMap [RegionMap, !Ref AWS::Region, AMI]
      InstanceType: t3.micro
      Tags:
        - Key: Name
          Value: otel-frontend
      KeyName: !Sub otel-observability-${AWS::Region}
      SecurityGroups:
        - !Ref FrontendInstanceSecurityGroup
      UserData:
        Fn::Base64: !Sub |
          #!/bin/bash
          sudo yum update -y
          echo ">> INSTALLING DOCKER"
          sudo yum install -y docker
          sudo service docker start
          sudo usermod -a -G docker ec2-user
          docker --version
          echo ">> INSTALLING DOCKER"
          sudo yum install -y git

  FrontendInstanceSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      Tags:
        - Key: App
          Value: otel-frontend
        - Key: Name
          Value: otel-frontend-sg
      GroupDescription: "otel-frontend security group"
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 80
          ToPort: 80
          CidrIp: 0.0.0.0/0
        - IpProtocol: tcp
          FromPort: 22
          ToPort: 22
          CidrIp: 0.0.0.0/0

  FilesServiceInstance:
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !FindInMap [RegionMap, !Ref AWS::Region, AMI]
      InstanceType: t3.micro
      KeyName: !Sub otel-observability-${AWS::Region}
      Tags:
        - Key: Name
          Value: otel-files-service
      IamInstanceProfile: !Ref FilesServiceInstanceProfile
      SecurityGroups:
        - !Ref FilesServiceInstanceSecurityGroup
      UserData:
        Fn::Base64: !Sub |
          #!/bin/bash
          sudo yum update -y
          sudo yum install -y docker
          sudo service docker start
          sudo usermod -a -G docker ec2-user
          docker --version
          sudo yum install -y git

  FilesServiceInstanceSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      Tags:
        - Key: App
          Value: otel-files-service
        - Key: Name
          Value: otel-files-service-sg
      GroupDescription: "otel-files-service security group"
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 80
          ToPort: 80
          CidrIp: 0.0.0.0/0
        - IpProtocol: tcp
          FromPort: 22
          ToPort: 22
          CidrIp: 0.0.0.0/0

  FilesServiceInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
        - !Ref FilesServiceInstanceRole

  FilesServiceInstanceRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub otel-observability-files-service-role-${AWS::Region}
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service: ec2.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: S3AccessPolicy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - s3:AbortMultipartUpload
                  - s3:ListMultipartUploadParts
                  - s3:ListBucketMultipartUploads
                  - s3:PutObject
                  - s3:GetObject
                Resource:
                  - !Sub arn:aws:s3:::${Bucket}/*
                  - !Sub arn:aws:s3:::${Bucket}
        - PolicyName: DynamoDBAccessPolicy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - dynamodb:GetItem
                  - dynamodb:PutItem
                  - dynamodb:UpdateItem
                  - dynamodb:DeleteItem
                  - dynamodb:Query
                Resource:
                  - !GetAtt DynamoFilesTable.Arn
                  - !Join [ "/", [ !GetAtt DynamoFilesTable.Arn, "*" ] ]
        - PolicyName: SQSAccessPolicy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - sqs:SendMessage
                Resource:
                  - !GetAtt FilesQueue.Arn

  AuthServiceInstance:
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !FindInMap [RegionMap, !Ref AWS::Region, AMI]
      InstanceType: t3.micro
      KeyName: !Sub otel-observability-${AWS::Region}
      IamInstanceProfile: !Ref AuthServiceInstanceProfile
      Tags:
        - Key: Name
          Value: otel-auth-service
      SecurityGroups:
        - !Ref AuthServiceInstanceSecurityGroup
      UserData:
        Fn::Base64: !Sub |
          #!/bin/bash
          sudo yum update -y
          sudo yum install -y docker
          sudo service docker start
          sudo usermod -a -G docker ec2-user
          docker --version
          sudo yum install -y git

  AuthServiceInstanceSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      Tags:
        - Key: App
          Value: otel-auth-service
        - Key: Name
          Value: otel-auth-service-sg
      GroupDescription: "otel-auth-service security group"
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 80
          ToPort: 80
          CidrIp: 0.0.0.0/0
        - IpProtocol: tcp
          FromPort: 22
          ToPort: 22
          CidrIp: 0.0.0.0/0

  AuthServiceInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
        - !Ref AuthServiceInstanceRole

  AuthServiceInstanceRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub otel-observability-auth-service-role-${AWS::Region}
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: ec2.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: DynamoDBAccessPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - dynamodb:GetItem
                  - dynamodb:PutItem
                  - dynamodb:UpdateItem
                  - dynamodb:DeleteItem
                  - dynamodb:Scan
                Resource:
                  - !GetAtt DynamoAuthTable.Arn
                  - !Join [ "/", [ !GetAtt DynamoAuthTable.Arn, "*" ] ]

  Bucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: !Ref BucketName
      CorsConfiguration:
        CorsRules:
          - AllowedHeaders: ['*']
            AllowedMethods: [PUT]
            AllowedOrigins: [!Sub 'http://${FrontendInstance.PublicIp}']
            ExposedHeaders: [ETag]
            MaxAge: 3600 # 1 hour (to avoid cors preflight requests)

  DynamoAuthTable:
    Type: AWS::DynamoDB::Table
    Properties:
      TableName: otel-observability-auth
      AttributeDefinitions:
        - AttributeName: token
          AttributeType: S
      KeySchema:
        - AttributeName: token
          KeyType: HASH
      BillingMode: PAY_PER_REQUEST
      Tags:
        - Key: Name
          Value: otel-observability-auth

  DynamoFilesTable:
    Type: AWS::DynamoDB::Table
    Properties:
      TableName: otel-observability-files
      AttributeDefinitions:
        - AttributeName: id
          AttributeType: S
        - AttributeName: data_type
          AttributeType: S
      KeySchema:
        - AttributeName: id
          KeyType: HASH # Partition key
      GlobalSecondaryIndexes:
        - IndexName: creation_datetime-index
          KeySchema:
            # data_type is an artificial attribute to "hack" dynamodb 
            # because its required add a has key
            - AttributeName: data_type
              KeyType: HASH
            # id is a ULID what is a string of 26 characters that can be ordered
            - AttributeName: id 
              KeyType: RANGE
          Projection:
            ProjectionType: ALL # All attributes in table
      BillingMode: PAY_PER_REQUEST
      Tags:
        - Key: Name
          Value: otel-observability-files

  FilesQueue:
    Type: AWS::SQS::Queue
    Properties:
      QueueName: otel-observability-files-queue
      VisibilityTimeout: 900 # 15 minutes

  PipelineFunctionRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub otel-observability-pipeline-function-role-${AWS::Region}
      AssumeRolePolicyDocument: 
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: SQSAccessPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - sqs:ReceiveMessage
                  - sqs:DeleteMessage
                  - sqs:GetQueueAttributes
                Resource: !GetAtt FilesQueue.Arn
        - PolicyName: DynamoDBAccessPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - dynamodb:GetItem
                  - dynamodb:UpdateItem
                Resource: !GetAtt DynamoFilesTable.Arn
        - PolicyName: RedshiftServerlessAccessPolicy
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - redshift-data:ExecuteStatement
                  - redshift-serverless:GetCredentials
                  - redshift-data:GetStatementResult
                  - redshift-data:DescribeStatement
                  - redshift-data:ListStatements
                Resource: '*'
          
  PipelineFunctionLogs:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: otel-observability-pipeline-function-logs

  PipelineFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: otel-observability-pipeline-function
      Handler: src.app.main
      Code:
        ZipFile: |
          # this code will be replaced after, it is just to create de lambda function
          def main(event, context):
            print("empty lambda function")
      Role: !GetAtt PipelineFunctionRole.Arn
      Runtime: python3.10
      Timeout: 900 # 15 minutes
      LoggingConfig:
        LogGroup: !Ref PipelineFunctionLogs
        LogFormat: Text

  PipelineLambdaSQSPermission:
    Type: AWS::Lambda::Permission
    Properties:
      FunctionName: !Ref PipelineFunction
      Action: "lambda:InvokeFunction"
      Principal: "sqs.amazonaws.com"
      SourceArn: !GetAtt FilesQueue.Arn

  PipelineCodeBuildRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub otel-observability-pipeline-codebuild-role-${AWS::Region}
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service: codebuild.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: LoadPipelineFunctionPolicy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - lambda:UpdateFunctionCode
                  - lambda:GetFunction
                  - lambda:UpdateFunctionConfiguration
                  - lambda:GetFunctionConfiguration
                Resource:
                  - !GetAtt PipelineFunction.Arn
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                Resource:
                  - "*"

  PipelineCodeBuildProject:
    Type: AWS::CodeBuild::Project
    Properties:
      Name: otel-observability-pipeline-codebuild-project
      Source:
        Type: GITHUB
        Location: https://github.com/CrissAlvarezH/otel-observability.git
        BuildSpec: apps/load-pipeline/buildspec.yml
      ServiceRole: !GetAtt PipelineCodeBuildRole.Arn
      Environment:
        ComputeType: BUILD_GENERAL1_SMALL
        Image: aws/codebuild/standard:5.0
        Type: LINUX_CONTAINER
        EnvironmentVariables:
          - Name: AWS_REGION
            Value: !Ref AWS::Region
          - Name: S3_BUCKET_NAME
            Value: !Ref Bucket
          - Name: LAMBDA_FUNCTION_NAME
            Value: !Ref PipelineFunction
          - Name: REDSHIFT_WORKGROUP
            Value: !Ref RedshiftServerlessWorkgroup
          - Name: REDSHIFT_DATABASE
            Value: otel-observability
          - Name: OTLP_COLLECTOR_ENDPOINT
            Value: !Sub "http://${ObservabilityBackendInstance.PublicIp}:4318/v1/traces"
      Artifacts:
        Type: NO_ARTIFACTS
      TimeoutInMinutes: 15

  SQSEventSourceMapping:
    Type: AWS::Lambda::EventSourceMapping
    Properties:
      BatchSize: 5 # number of messages to process per invocation
      EventSourceArn: !GetAtt FilesQueue.Arn
      FunctionName: !Ref PipelineFunction
      Enabled: True

  RedshiftServerlessNamespace:
    Type: AWS::RedshiftServerless::Namespace
    Properties:
      NamespaceName: "otel-observability-namespace"
      AdminUsername: "otel"
      AdminUserPassword: "OtelObservability123"
      DbName: "otel-observability"
      DefaultIamRoleArn: !GetAtt RedshiftServerlessIamRole.Arn
      IamRoles:
        - !GetAtt RedshiftServerlessIamRole.Arn

  RedshiftServerlessWorkgroup:
    Type: AWS::RedshiftServerless::Workgroup
    Properties:
      WorkgroupName: "otel-observability-workgroup"
      NamespaceName: !Ref RedshiftServerlessNamespace
      BaseCapacity: 8 
      PubliclyAccessible: true # to allow copy from s3
      EnhancedVpcRouting: true

  RedshiftServerlessIamRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub otel-observability-redshift-serverless-role-${AWS::Region}
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service: 
                - redshift-serverless.amazonaws.com
                - redshift.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: "RedshiftServerlessS3Access"
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - s3:GetObject
                  - s3:ListBucket
                Resource:
                  - !Sub arn:aws:s3:::${Bucket}/*
                  - !Sub arn:aws:s3:::${Bucket}

  ObservabilityBackendInstance:
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !FindInMap [RegionMap, !Ref AWS::Region, AMI]
      InstanceType: t3.medium
      KeyName: !Sub otel-observability-${AWS::Region}
      Tags:
        - Key: Name
          Value: observability-backend-service
      SecurityGroups:
        - !Ref ObservabilityBackendInstanceSecurityGroup
      UserData:
        Fn::Base64: !Sub |
          #!/bin/bash
          sudo yum update -y
          sudo yum install -y docker
          sudo service docker start
          sudo usermod -a -G docker ec2-user
          docker --version
          sudo yum install -y git

          sudo curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
          sudo chmod +x /usr/local/bin/docker-compose
          docker-compose --version

  ObservabilityBackendInstanceSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      Tags:
        - Key: Name
          Value: observability-backend-sg
      GroupDescription: "observability-backend-service security group"
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 16686
          ToPort: 16686
          CidrIp: 0.0.0.0/0
        - IpProtocol: tcp
          FromPort: 22
          ToPort: 22
          CidrIp: 0.0.0.0/0
        - IpProtocol: tcp
          FromPort: 4317
          ToPort: 4317
          CidrIp: 0.0.0.0/0
        - IpProtocol: tcp
          FromPort: 4318
          ToPort: 4318
          CidrIp: 0.0.0.0/0


Outputs:
  FrontendInstancePublicIp:
    Description: Public IP of the frontend instance
    Value: !GetAtt FrontendInstance.PublicIp

  FilesServiceInstancePublicIp:
    Description: Public IP of the files service instance
    Value: !GetAtt FilesServiceInstance.PublicIp

  AuthServiceInstancePublicIp:
    Description: Public IP of the auth service instance
    Value: !GetAtt AuthServiceInstance.PublicIp

  ObservabilityBackendInstancePublicIp:
    Description: Public IP of the observability backend instance
    Value: !GetAtt ObservabilityBackendInstance.PublicIp

  FilesQueueUrl:
    Description: URL of the files queue
    Value: !GetAtt FilesQueue.QueueUrl