Skip to content

Installation in AWS ECS

Jump to bottom
Daria Kharlan edited this page Nov 15, 2021 · 4 revisions

Cloudformation template example

AWSTemplateFormatVersion: 2010-09-09
Parameters:
  VpcId:
    Type: AWS::EC2::VPC::Id
  SubnetId:
    Type: AWS::EC2::Subnet::Id
    Description: Must have internet access to pull the images from dockerhub
  AgentImage:
    Type: String
    Default: docker.io/anodot/daria:3.22.0
  AgentCpu:
    Type: Number
    Default: 2048
  AgentMemory:
    Type: Number
    Default: 4096
  StreamsetsImage:
    Type: String
    Default: docker.io/anodot/streamsets:3.22.0
  StreamsetsCpu:
    Type: Number
    Default: 4096
  StreamsetsMemory:
    Type: Number
    Default: 8192
  StreamsetsJavaOpts:
    Type: String
    Default: " -Xmx6144m -Xms6144m -server"
Resources:
  AgentService:
    DependsOn:
    - DcService
    Properties:
      Cluster:
        Fn::GetAtt:
        - Cluster
        - Arn
      DeploymentConfiguration:
        MaximumPercent: 200
        MinimumHealthyPercent: 100
      DeploymentController:
        Type: ECS
      DesiredCount: 1
      LaunchType: FARGATE
      NetworkConfiguration:
        AwsvpcConfiguration:
          AssignPublicIp: ENABLED
          SecurityGroups:
          - Ref: DefaultNetwork
          Subnets:
          - Ref: SubnetId
      PlatformVersion: 1.4.0
      PropagateTags: SERVICE
      SchedulingStrategy: REPLICA
      ServiceRegistries:
      - RegistryArn:
          Fn::GetAtt:
          - AgentServiceDiscoveryEntry
          - Arn
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.service
        Value: agent
      TaskDefinition:
        Ref: AgentTaskDefinition
      EnableExecuteCommand: true
    Type: AWS::ECS::Service
  AgentServiceDiscoveryEntry:
    Properties:
      Description: '"agent" service discovery entry in Cloud Map'
      DnsConfig:
        DnsRecords:
        - TTL: 60
          Type: A
        RoutingPolicy: MULTIVALUE
      HealthCheckCustomConfig:
        FailureThreshold: 1
      Name: agent
      NamespaceId:
        Ref: CloudMap
    Type: AWS::ServiceDiscovery::Service
  AgentTaskDefinition:
    Properties:
      ContainerDefinitions:
      - Command:
        - us-east-1.compute.internal
        - agent-ecs.local
        Essential: false
        Image: docker/ecs-searchdomain-sidecar:1.0
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: agent-ecs
        Name: Agent_ResolvConf_InitContainer
      - DependsOn:
        - Condition: SUCCESS
          ContainerName: Agent_ResolvConf_InitContainer
        Essential: true
        Image:
          Ref: AgentImage
        LinuxParameters: {}
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: agent-ecs
        Name: agent
        Environment:
        - Name: LISTEN_PORT
          Value: 8080
      Cpu:
        Ref: AgentCpu
      ExecutionRoleArn:
        Ref: AgentTaskExecutionRole
      TaskRoleArn:
        Ref: AgentTaskExecutionRole
      Family: agent-ecs-agent
      Memory:
        Ref: AgentMemory
      NetworkMode: awsvpc
      RequiresCompatibilities:
      - FARGATE
    Type: AWS::ECS::TaskDefinition
  AgentTaskExecutionRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Condition: {}
          Effect: Allow
          Principal:
            Service: ecs-tasks.amazonaws.com
        Version: 2012-10-17
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
      - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
      - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.service
        Value: agent
    Type: AWS::IAM::Role
  AgentdataAccessPoint:
    Properties:
      AccessPointTags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.volume
        Value: agent-data
      - Key: Name
        Value: agent-ecs_agent-data
      FileSystemId:
        Ref: AgentdataFilesystem
    Type: AWS::EFS::AccessPoint
  AgentdataFilesystem:
    DeletionPolicy: Retain
    Properties:
      Encrypted: true
      FileSystemTags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.volume
        Value: agent-data
      - Key: Name
        Value: agent-ecs_agent-data
    Type: AWS::EFS::FileSystem
  AgentdataNFSMountTarget:
    Properties:
      FileSystemId:
        Ref: AgentdataFilesystem
      SecurityGroups:
      - Ref: DefaultNetwork
      SubnetId:
        Ref: SubnetId
    Type: AWS::EFS::MountTarget
  CloudMap:
    Properties:
      Description: Service Map for Docker Compose project agent-ecs
      Name: agent-ecs.local
      Vpc:
        Ref: VpcId
    Type: AWS::ServiceDiscovery::PrivateDnsNamespace
  Cluster:
    Properties:
      ClusterName: agent-ecs
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
    Type: AWS::ECS::Cluster
  DbService:
    DependsOn:
    - AgentdataNFSMountTarget
    Properties:
      Cluster:
        Fn::GetAtt:
        - Cluster
        - Arn
      DeploymentConfiguration:
        MaximumPercent: 200
        MinimumHealthyPercent: 100
      DeploymentController:
        Type: ECS
      DesiredCount: 1
      LaunchType: FARGATE
      NetworkConfiguration:
        AwsvpcConfiguration:
          AssignPublicIp: ENABLED
          SecurityGroups:
          - Ref: DefaultNetwork
          Subnets:
          - Ref: SubnetId
      PlatformVersion: 1.4.0
      PropagateTags: SERVICE
      SchedulingStrategy: REPLICA
      ServiceRegistries:
      - RegistryArn:
          Fn::GetAtt:
          - DbServiceDiscoveryEntry
          - Arn
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.service
        Value: db
      TaskDefinition:
        Ref: DbTaskDefinition
    Type: AWS::ECS::Service
  DbServiceDiscoveryEntry:
    Properties:
      Description: '"db" service discovery entry in Cloud Map'
      DnsConfig:
        DnsRecords:
        - TTL: 60
          Type: A
        RoutingPolicy: MULTIVALUE
      HealthCheckCustomConfig:
        FailureThreshold: 1
      Name: db
      NamespaceId:
        Ref: CloudMap
    Type: AWS::ServiceDiscovery::Service
  DbTaskDefinition:
    Properties:
      ContainerDefinitions:
      - Command:
        - us-east-1.compute.internal
        - agent-ecs.local
        Essential: false
        Image: docker/ecs-searchdomain-sidecar:1.0
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: agent-ecs
        Name: Db_ResolvConf_InitContainer
      - DependsOn:
        - Condition: SUCCESS
          ContainerName: Db_ResolvConf_InitContainer
        Environment:
        - Name: POSTGRES_DB
          Value: agent
        - Name: POSTGRES_PASSWORD
          Value: agent
        - Name: POSTGRES_USER
          Value: agent
        Essential: true
        Image: docker.io/library/postgres:latest@sha256:2b87b5bb55589540f598df6ec5855e5c15dd13628230a689d46492c1d433c4df
        LinuxParameters: {}
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: agent-ecs
        MountPoints:
        - ContainerPath: /var/lib/postgresql/data
          SourceVolume: agent-data
        Name: db
      Cpu: "2048"
      ExecutionRoleArn:
        Ref: DbTaskExecutionRole
      Family: agent-ecs-db
      Memory: "4096"
      NetworkMode: awsvpc
      RequiresCompatibilities:
      - FARGATE
      TaskRoleArn:
        Ref: DbTaskRole
      Volumes:
      - EFSVolumeConfiguration:
          AuthorizationConfig:
            AccessPointId:
              Ref: AgentdataAccessPoint
            IAM: ENABLED
          FilesystemId:
            Ref: AgentdataFilesystem
          TransitEncryption: ENABLED
        Name: agent-data
    Type: AWS::ECS::TaskDefinition
  DbTaskExecutionRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Condition: {}
          Effect: Allow
          Principal:
            Service: ecs-tasks.amazonaws.com
        Version: 2012-10-17
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
      - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.service
        Value: db
    Type: AWS::IAM::Role
  DbTaskRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Condition: {}
          Effect: Allow
          Principal:
            Service: ecs-tasks.amazonaws.com
        Version: 2012-10-17
      Policies:
      - PolicyDocument:
          Statement:
          - Action:
            - elasticfilesystem:ClientMount
            - elasticfilesystem:ClientWrite
            - elasticfilesystem:ClientRootAccess
            Condition:
              StringEquals:
                elasticfilesystem:AccessPointArn:
                  Ref: AgentdataAccessPoint
            Effect: Allow
            Principal: {}
            Resource:
            - Fn::GetAtt:
              - AgentdataFilesystem
              - Arn
          Version: 2012-10-17
        PolicyName: DbAgentdataVolumeMountPolicy
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.service
        Value: db
    Type: AWS::IAM::Role
  DcService:
    DependsOn:
    - SdcdataNFSMountTarget
    Properties:
      Cluster:
        Fn::GetAtt:
        - Cluster
        - Arn
      DeploymentConfiguration:
        MaximumPercent: 200
        MinimumHealthyPercent: 100
      DeploymentController:
        Type: ECS
      DesiredCount: 1
      LaunchType: FARGATE
      NetworkConfiguration:
        AwsvpcConfiguration:
          AssignPublicIp: ENABLED
          SecurityGroups:
          - Ref: DefaultNetwork
          Subnets:
          - Ref: SubnetId
      PlatformVersion: 1.4.0
      PropagateTags: SERVICE
      SchedulingStrategy: REPLICA
      ServiceRegistries:
      - RegistryArn:
          Fn::GetAtt:
          - DcServiceDiscoveryEntry
          - Arn
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.service
        Value: dc
      TaskDefinition:
        Ref: DcTaskDefinition
    Type: AWS::ECS::Service
  DcServiceDiscoveryEntry:
    Properties:
      Description: '"dc" service discovery entry in Cloud Map'
      DnsConfig:
        DnsRecords:
        - TTL: 60
          Type: A
        RoutingPolicy: MULTIVALUE
      HealthCheckCustomConfig:
        FailureThreshold: 1
      Name: dc
      NamespaceId:
        Ref: CloudMap
    Type: AWS::ServiceDiscovery::Service
  DcTaskDefinition:
    Properties:
      ContainerDefinitions:
      - Command:
        - us-east-1.compute.internal
        - agent-ecs.local
        Essential: false
        Image: docker/ecs-searchdomain-sidecar:1.0
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: agent-ecs
        Name: Dc_ResolvConf_InitContainer
      - DependsOn:
        - Condition: SUCCESS
          ContainerName: Dc_ResolvConf_InitContainer
        Environment:
        - Name: DC_CONF_RUNNER_THREAD_POOL_SIZE
          Value: "500"
        - Name: SDC_JAVA_OPTS
          Value:
            Ref: StreamsetsJavaOpts
        Essential: true
        Image:
          Ref: StreamsetsImage
        LinuxParameters: {}
        LogConfiguration:
          LogDriver: awslogs
          Options:
            awslogs-group:
              Ref: LogGroup
            awslogs-region:
              Ref: AWS::Region
            awslogs-stream-prefix: agent-ecs
        MountPoints:
        - ContainerPath: /data
          SourceVolume: sdc-data
        Name: dc
        Ulimits:
        - HardLimit: 32768
          Name: nofile
          SoftLimit: 32768
      Cpu:
        Ref: StreamsetsCpu
      ExecutionRoleArn:
        Ref: DcTaskExecutionRole
      Family: agent-ecs-dc
      Memory:
        Ref: StreamsetsMemory
      NetworkMode: awsvpc
      RequiresCompatibilities:
      - FARGATE
      TaskRoleArn:
        Ref: DcTaskRole
      Volumes:
      - EFSVolumeConfiguration:
          AuthorizationConfig:
            AccessPointId:
              Ref: SdcdataAccessPoint
            IAM: ENABLED
          FilesystemId:
            Ref: SdcdataFilesystem
          TransitEncryption: ENABLED
        Name: sdc-data
    Type: AWS::ECS::TaskDefinition
  DcTaskExecutionRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Condition: {}
          Effect: Allow
          Principal:
            Service: ecs-tasks.amazonaws.com
        Version: 2012-10-17
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
      - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.service
        Value: dc
    Type: AWS::IAM::Role
  DcTaskRole:
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action:
          - sts:AssumeRole
          Condition: {}
          Effect: Allow
          Principal:
            Service: ecs-tasks.amazonaws.com
        Version: 2012-10-17
      Policies:
      - PolicyDocument:
          Statement:
          - Action:
            - elasticfilesystem:ClientMount
            - elasticfilesystem:ClientWrite
            - elasticfilesystem:ClientRootAccess
            Condition:
              StringEquals:
                elasticfilesystem:AccessPointArn:
                  Ref: SdcdataAccessPoint
            Effect: Allow
            Principal: {}
            Resource:
            - Fn::GetAtt:
              - SdcdataFilesystem
              - Arn
          Version: 2012-10-17
        PolicyName: DcSdcdataVolumeMountPolicy
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.service
        Value: dc
    Type: AWS::IAM::Role
  DefaultNetwork:
    Properties:
      GroupDescription: agent-ecs Security Group for default network
      Tags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.network
        Value: agent-ecs_default
      VpcId:
        Ref: VpcId
    Type: AWS::EC2::SecurityGroup
  DefaultNetworkIngress:
    Properties:
      Description: Allow communication within network default
      GroupId:
        Ref: DefaultNetwork
      IpProtocol: "-1"
      SourceSecurityGroupId:
        Ref: DefaultNetwork
    Type: AWS::EC2::SecurityGroupIngress
  LogGroup:
    Properties:
      LogGroupName: /docker-compose/agent-ecs
    Type: AWS::Logs::LogGroup
  SdcdataAccessPoint:
    Properties:
      AccessPointTags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.volume
        Value: sdc-data
      - Key: Name
        Value: agent-ecs_sdc-data
      FileSystemId:
        Ref: SdcdataFilesystem
      PosixUser: 
        Gid: 20159
        Uid: 20159
      RootDirectory:
        Path: '/sdc'
        CreationInfo: 
          OwnerGid: 20159
          OwnerUid: 20159
          Permissions: 775
    Type: AWS::EFS::AccessPoint
  SdcdataFilesystem:
    DeletionPolicy: Retain
    Properties:
      Encrypted: true
      FileSystemTags:
      - Key: com.docker.compose.project
        Value: agent-ecs
      - Key: com.docker.compose.volume
        Value: sdc-data
      - Key: Name
        Value: agent-ecs_sdc-data
    Type: AWS::EFS::FileSystem
  SdcdataNFSMountTarget:
    Properties:
      FileSystemId:
        Ref: SdcdataFilesystem
      SecurityGroups:
      - Ref: DefaultNetwork
      SubnetId:
        Ref: SubnetId
    Type: AWS::EFS::MountTarget

The required parameters are VpcId and SubnetId (the subnet should have internet access to pull images). You can also adjust agent and dc container CPU and memory in case you expect high eps or create a lot of pipelines (2vCPU can handle around 1k EPS, 8GB Memory ~30 pipelines). It is recommended not to create more than 70 pipelines in one streamsets instance.

Access the agent

aws ecs execute-command --cluster agent-ecs --task {TASKID} --container agent --command "/bin/bash" --interactive

Replace {TASKID} with the id of the task created for the agent service

Configuring the agent

  1. Add streamsets instance
/usr/src/app# agent streamsets add
Enter streamsets url: http://dc:18630
Username [admin]: 
Password [admin]: 
2021-11-09 13:17:06,331 - INFO - Get pipelines
Agent external URL [http://anodot-agent]: http://agent:8080

Next steps are described here Basic flow

Clone this wiki locally