diff --git a/Dockerfile b/Dockerfile index 2595030..225f085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,26 @@ FROM public.ecr.aws/lambda/python:3.12 ENV LAMBDA_TASK_ROOT=/var/runtime -# Copy Starfleet over (just a quick and dirty sample, not the cleanest): +# Copy Starfleet over: COPY ./ ${LAMBDA_TASK_ROOT}/starfleet +# Perform cleanup of things the Docker doens't need: +RUN rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/tests && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/.tox && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/.pytest_cache && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/deploy && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/site && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/.kubelr && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/build && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/.coverage && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/*.ini && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/*.yaml && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/*.md && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/sample_samconfig.toml && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/Dockerfile && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/venv && \ + rm -Rf ${LAMBDA_TASK_ROOT}/starfleet/env + # Install the specified packages: RUN cd ${LAMBDA_TASK_ROOT}/starfleet && \ pip install . && \ diff --git a/mkdocs.yml b/mkdocs.yml index 5be9fe2..b78778a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -38,9 +38,6 @@ theme: plugins: - search - blog - - gen-files: - scripts: - - mkdocs/gen_ref_pages.py - literate-nav: nav_file: SUMMARY.md - mkdocstrings @@ -87,6 +84,7 @@ nav: - Set up IAM: installation/IAM.md - Prepare Configuration: installation/PrepareConfiguration.md - Prepare Payload Templates: installation/PreparePayload.md + - Set up ECR: installation/SetupECR.md - Run AWS SAM: installation/RunSAM.md - Make the Account Index: installation/MakeAccountIndex.md - Wrapping Up: installation/WrapUp.md @@ -135,4 +133,3 @@ nav: - Extras: - Overview: Links.md - Attributions: Attributions.md - - Code Reference: reference/ diff --git a/mkdocs/Links.md b/mkdocs/Links.md index 8d5daad..89487ba 100644 --- a/mkdocs/Links.md +++ b/mkdocs/Links.md @@ -6,6 +6,3 @@ See [the attributions](Attributions.md) page for additional attributions for thi ## GitHub 1. Main repository link: [https://github.com/gemini-oss/starfleet](https://github.com/gemini-oss/starfleet) 1. Other cool Gemini OSS projects: [https://github.com/gemini-oss](https://github.com/gemini-oss) - -## Code Reference -We also have the [code to reference here](/starfleet/reference/starfleet/). diff --git a/mkdocs/blog/posts/ecr.md b/mkdocs/blog/posts/ecr.md new file mode 100644 index 0000000..d78f22d --- /dev/null +++ b/mkdocs/blog/posts/ecr.md @@ -0,0 +1,58 @@ +--- +date: 2024-04-08 +authors: + - mikegrima +categories: + - General Announcement +--- + +# ECR/Dockerized Lambda Support +We made a small update to the docs to highlight deploying Starfleet with a Dockerized Lambda. This was necessary as Starfleet's dependencies made it larger than the `.zip` file size limit for Lambda. + +The main steps to convert to this from a non-ECR set up is to: + +1. Create the Private ECR repo in the same account/region you have Starfleet deployed in. +2. Update the `samconfig.toml` file. +3. Update the SAM template. +4. Re-build and deploy. + +The Dockerized Lambda is built with the included `Dockerfile`, which should have everything needed and ready to go. You can test that the container builds by running `docker build .` from within the Starfleet directory. + +## Update SAM Config +Once you create your ECR Repo, you then need to update your `samconfig.toml` file to include the line: +``` +image_repository = "ACCOUNT_ID.dkr.ecr.REGION.amazonaws.com/REPONAME" +``` +... under your `.deploy.parameters` sections. + +## Update your SAM Template +The SAM template also needs to be updated. Included now are 2 sample SAM Templates: `test_sam_template.yaml`, which is the ECR sample template, and `test_sam_template_NO_ECR.yaml`, which is the original non-ECR version. + +You would need to update all the function entires to change it from: +```yaml + StarbaseEventBridgeFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ./src + Handler: starfleet.starbase.entrypoints.eventbridge_timed_lambda_handler + Runtime: python3.12 +``` +over to: + +```yaml + StarbaseEventBridgeFunction: + Type: AWS::Serverless::Function + Metadata: + DockerTag: starfleet + DockerContext: ./ + Dockerfile: Dockerfile + Properties: + PackageType: Image + ImageConfig: + Command: + - starfleet.starbase.entrypoints.eventbridge_timed_lambda_handler +``` + +Update all the Lambda definitions to include the `Metadata` section as shown above (this is the same for all the functions) and update the `Properties` section to have the fields above. Remove the old fields. Note: the `Command` field is where you place the Lambda handler path - this is unique to each Lambda function. + +For more information see [the ECR setup documentation](../../installation/SetupECR.md) and also [AWS's documentation here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html). diff --git a/mkdocs/developerGuide/SAMConfiguration.md b/mkdocs/developerGuide/SAMConfiguration.md index 50167bb..923f64c 100644 --- a/mkdocs/developerGuide/SAMConfiguration.md +++ b/mkdocs/developerGuide/SAMConfiguration.md @@ -1,6 +1,6 @@ # AWS SAM Configuration -Last but certainly not least is the AWS SAM specification. We provide a sample AWS SAM template to make things easy to deploy. +Last but certainly not least is the AWS SAM specification. We provide a sample AWS SAM template to make things easy to deploy. We recommend setting up ECR to have a Dockerized Lambda function since the Starfleet dependencies may blow past the Lambda filesize limit. All examples below assume we are using ECR. At a minimum, you'll need to have a defined Lambda function, IAM permissions (typically to assume a role -- with a target role that permits the access), and that's basically it. @@ -28,10 +28,15 @@ AccountIndexGeneratorQueue: # The Lambda function: AccountIndexGenerator: Type: AWS::Serverless::Function + Metadata: + DockerTag: starfleet + DockerContext: ./ + Dockerfile: Dockerfile Properties: - CodeUri: ./src - Handler: starfleet.worker_ships.plugins.account_index_generator.ship.lambda_handler - Runtime: python3.10 + PackageType: Image + ImageConfig: + Command: + - starfleet.worker_ships.plugins.account_index_generator.ship.lambda_handler Architectures: - arm64 MemorySize: 256 @@ -64,10 +69,15 @@ StarbaseFanoutFunction: Type: AWS::Serverless::Function DependsOn: - AccountIndexGeneratorQueue + Metadata: + DockerTag: starfleet + DockerContext: ./ + Dockerfile: Dockerfile Properties: - CodeUri: ./src - Handler: starfleet.starbase.entrypoints.fanout_payload_lambda_handler - Runtime: python3.10 + PackageType: Image + ImageConfig: + Command: + - starfleet.starbase.entrypoints.fanout_payload_lambda_handler Architectures: - arm64 Events: @@ -134,7 +144,7 @@ region = "REPLACEME" confirm_changeset = true capabilities = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"] # Important -- you need these capabilities defined since this creates IAM roles parameter_overrides = "EnvironmentName=\"TEST\"" -image_repositories = [] +image_repository = "REPLACE ME - IF YOU ARE NOT USING ECR DELETE THIS LINE" [TEST.validate.parameters] region = "REPLACEME" @@ -155,7 +165,7 @@ region = "REPLACEME" confirm_changeset = true capabilities = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"] # Important -- you need these capabilities defined since this creates IAM roles parameter_overrides = "EnvironmentName=\"PROD\"" -image_repositories = [] +image_repository = "REPLACE ME - IF YOU ARE NOT USING ECR DELETE THIS LINE" [PROD.validate.parameters] region = "REPLACEME" diff --git a/mkdocs/gen_ref_pages.py b/mkdocs/gen_ref_pages.py deleted file mode 100644 index 3e7386e..0000000 --- a/mkdocs/gen_ref_pages.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Generate the code reference pages.""" - -## This is from https://mkdocstrings.github.io/recipes/#generate-pages-on-the-fly - -from pathlib import Path - -import mkdocs_gen_files - -nav = mkdocs_gen_files.Nav() - -for path in sorted(Path("src").rglob("*.py")): # - module_path = path.relative_to("src").with_suffix("") # - doc_path = path.relative_to("src").with_suffix(".md") # - full_doc_path = Path("reference", doc_path) # - - parts = list(module_path.parts) - - if parts[-1] == "__init__": # - parts = parts[:-1] - doc_path = doc_path.with_name("index.md") - full_doc_path = full_doc_path.with_name("index.md") - elif parts[-1] == "__main__": - continue - - nav[parts] = doc_path.as_posix() # - - with mkdocs_gen_files.open(full_doc_path, "w") as fd: # - identifier = ".".join(parts) # - print("::: " + identifier, file=fd) # - - # mkdocs_gen_files.set_edit_path(full_doc_path, path) # - mkdocs_gen_files.set_edit_path(full_doc_path, Path("../") / path) # per the docs - -with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file: # - nav_file.writelines(nav.build_literate_nav()) # diff --git a/mkdocs/installation/RunSAM.md b/mkdocs/installation/RunSAM.md index bd1282b..23e8651 100644 --- a/mkdocs/installation/RunSAM.md +++ b/mkdocs/installation/RunSAM.md @@ -6,11 +6,17 @@ Now you are ready to run AWS SAM and get the infrastructure components deployed! This section could use some assistance from the community for simplifying the installation process and documentation. We would love for assistance on the creation of things like an installation script, for example that would set all of this up for you. ## The Test Template -We include a file called `test_sam_template.yaml`, so you can later clone it for production with production specific values in. The commands below will assume that the template is named `test_sam_template.yaml`. +We include a file called `test_sam_template.yaml` (for ECR) and `test_sam_template_NO_ECR.yaml`(for non-ECR), so you can later clone it for production with production specific values in. The commands below will assume that the template is named `test_sam_template.yaml`. ## Build Starfleet To build the code, in the main checked out directory you will run: +If you are using ECR: +```bash +sam build --template-file test_sam_template.yaml --parameter-overrides ParameterKey=EnvironmentName,ParameterValue=TEST +``` + +If you are not using ECR: ```bash sam build --use-container --template-file test_sam_template.yaml --parameter-overrides ParameterKey=EnvironmentName,ParameterValue=TEST ``` @@ -59,6 +65,7 @@ We need to update the SAM configuration, _manually_, to allow it to create IAM r [TEST.deploy.parameters] # ... capabilities = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"] +image_repository = "YOUR_ACCOUNT_ID.dkr.ecr.YOUR_REGION.amazonaws.com/REPO_NAME -- if you are using ECR" [TEST.validate.parameters] region = "YOUR_REGION_HERE" @@ -137,6 +144,7 @@ At this point you will have the following: - [x] Starfleet worker IAM roles deployed everywhere - [x] The `configuration.yaml` file in `src/starfleet/configuration_files` modified with values unique to your environment - [x] A payload template (not stored as a configuration file) in a different place than your configuration that describes what the Starfleet Account Index Generator is supposed to do +- [x] An optional ECR Repository set up to make dockerized Lambda builds - [x] And Now: AWS SAM: - [x] SAM's administrative resources deployed - [x] SAM's TEST deployment configuration all set up diff --git a/mkdocs/installation/SetupECR.md b/mkdocs/installation/SetupECR.md new file mode 100644 index 0000000..7d6796c --- /dev/null +++ b/mkdocs/installation/SetupECR.md @@ -0,0 +1,52 @@ +# Set Up Private ECR (Elastic Container Registry) + +This is not required but highly recommended as the Starfleet package may be too big for Lambda to accept it as a `.zip` file. As such, we recommend setting up Starfleet with a Dockerized Lambda hosted in a private ECR. + +The steps to do this will largely involve setting up ECR, which you can find in [the AWS documentation here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html). + +When creating your registry please adhere to the following: + +1. You are creating a **PRIVATE** registry. **DO NOT MAKE THIS PUBLIC!** +2. This must reside in the same account and region that you are deploying Starfleet to +3. Recommend that you set up tag immutability +4. Recommend that you set up a lifecycle policy to expire older images (recommend to expire any `Image count more than 3`) + +!!! danger "Public Access" + DO NOT make your ECR repository public! + +Once you have your ECR registry set up, there will be a URL associated with it. That URL is the where your image will be pushed to. You will need that in the next steps for AWS SAM. + +!!! note "Note: ECR URL" + You will need the ECR URL for the next sections. This is in the format of `aws_account_id.dkr.ecr.region.amazonaws.com` + +## Note about Dockerizing Starfleet +Included in the main code base is a `Dockerfile` that is intended to build Starfleet. This file should be all good to go. This is configured to use the AWS official Lambda docker container for a specific Python version. This also: + +1. Copies the Starfleet code to the container in the `var/runtime` directory, which is where Lambda wants the code to live. +2. Cleans up some unnecessary files. +2. It runs `pip install` to install the Starfleet packages +3. Lastly, and _very importantly_ it **removes** the built-in `boto3` and `botocore` packages that is shipped by default with the container. This is necessary because Lambda will prefer to use the preloaded version of boto instead of the Starfleet packaged one. The pre-loaded version can be out of date and can result in very bizarre errors with unsupported boto APIs - this is because you may be using boto features that are newer than the included boto packages. Thus, we remove them so only the Starfleet included boto packages are there. + +For more information about Dockerizing Lambda functions, [see the docs here](https://docs.aws.amazon.com/lambda/latest/dg/python-image.html). + +## Continuing + +At this point you will have the following: + +- [x] Enable AWS Organizations if you haven't already done so and move some accounts into it +- [x] Pick out an AWS account for deploying a testing version of Starfleet +- [x] Work on getting a read-only Starfleet IAM role deployed with the permissions outlined above in all your AWS accounts. This role is _not_ very permissive and is only able to describe the enabled regions for an account. + - [x] In the organization root, it has permissions to list the Organizations accounts. + - [x] If you use StackSets then you need to manually make the role in the org root since StackSets won't operate in the org root. + - [x] Important: Make sure that you have some IAM principal that you can use locally that can assume all these roles. This will be needed to run the Starfleet CLI. If you use AWS SSO, then use the ARN for the permissions set provisioned administrative role in the Starfleet account. See the note above for an example. +- [x] AWS Account identified for deployment +- [x] Starfleet worker IAM roles deployed everywhere +- [x] The `configuration.yaml` file in `src/starfleet/configuration_files` modified with values unique to your environment +- [x] A payload template (not stored as a configuration file) in a different place than your configuration that describes what the Starfleet Account Index Generator is supposed to do +- [x] An optional ECR Repository set up to make dockerized Lambda builds +- [x] And Now: AWS SAM: + - [x] SAM's administrative resources deployed + - [x] SAM's TEST deployment configuration all set up + - [x] Starfleet deployed in your environment + +While it's now deployed, it won't work without an Account Index. The next section describes how to get it set up. diff --git a/sample_samconfig.toml b/sample_samconfig.toml index 3f192ce..a32e9fa 100644 --- a/sample_samconfig.toml +++ b/sample_samconfig.toml @@ -9,7 +9,7 @@ region = "REPLACEME" confirm_changeset = true capabilities = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"] # Important -- you need these capabilities defined since this creates IAM roles parameter_overrides = "EnvironmentName=\"TEST\"" -image_repositories = [] +image_repository = "REPLACE ME - IF YOU ARE NOT USING ECR DELETE THIS LINE" [TEST.validate.parameters] region = "REPLACEME" @@ -31,7 +31,7 @@ region = "REPLACEME" confirm_changeset = true capabilities = ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"] # Important -- you need these capabilities defined since this creates IAM roles parameter_overrides = "EnvironmentName=\"PROD\"" -image_repositories = [] +image_repository = "REPLACE ME - IF YOU ARE NOT USING ECR DELETE THIS LINE" [PROD.validate.parameters] region = "REPLACEME" diff --git a/test_sam_template.yaml b/test_sam_template.yaml index 5dc42ac..2fb7f47 100644 --- a/test_sam_template.yaml +++ b/test_sam_template.yaml @@ -1,3 +1,4 @@ +# This is the ECR version of the SAM template: AWSTemplateFormatVersion: '2010-09-09' Transform: AWS::Serverless-2016-10-31 Description: > @@ -126,10 +127,15 @@ Resources: # The Starbase EventBridge listener: StarbaseEventBridgeFunction: Type: AWS::Serverless::Function + Metadata: + DockerTag: starfleet + DockerContext: ./ + Dockerfile: Dockerfile Properties: - CodeUri: ./src - Handler: starfleet.starbase.entrypoints.eventbridge_timed_lambda_handler - Runtime: python3.12 + PackageType: Image + ImageConfig: + Command: + - starfleet.starbase.entrypoints.eventbridge_timed_lambda_handler Architectures: - arm64 Events: @@ -223,10 +229,15 @@ Resources: AccountIndexGenerator: Type: AWS::Serverless::Function + Metadata: + DockerTag: starfleet + DockerContext: ./ + Dockerfile: Dockerfile Properties: - CodeUri: ./src - Handler: starfleet.worker_ships.plugins.account_index_generator.ship.lambda_handler - Runtime: python3.12 + PackageType: Image + ImageConfig: + Command: + - starfleet.worker_ships.plugins.account_index_generator.ship.lambda_handler Architectures: - arm64 MemorySize: 256 @@ -265,10 +276,15 @@ Resources: # # AWSConfigWorker: # Type: AWS::Serverless::Function +# Metadata: +# DockerTag: starfleet +# DockerContext: ./ +# Dockerfile: Dockerfile # Properties: -# CodeUri: ./src -# Handler: starfleet.worker_ships.plugins.aws_config.ship.lambda_handler -# Runtime: python3.12 +# PackageType: Image +# ImageConfig: +# Command: +# - starfleet.worker_ships.plugins.aws_config.ship.lambda_handler # Architectures: # - arm64 # MemorySize: 128 @@ -305,10 +321,15 @@ Resources: # # GitHubSyncWorker: # Type: AWS::Serverless::Function +# Metadata: +# DockerTag: starfleet +# DockerContext: ./ +# Dockerfile: Dockerfile # Properties: -# CodeUri: ./src -# Handler: starfleet.worker_ships.plugins.github_sync.ship.lambda_handler -# Runtime: python3.12 +# PackageType: Image +# ImageConfig: +# Command: +# - starfleet.worker_ships.plugins.github_sync.ship.lambda_handler # Architectures: # - arm64 # MemorySize: 128 @@ -346,10 +367,15 @@ Resources: # # IamRoleWorker: # Type: AWS::Serverless::Function +# Metadata: +# DockerTag: starfleet +# DockerContext: ./ +# Dockerfile: Dockerfile # Properties: -# CodeUri: ./starfleet/src -# Handler: starfleet.worker_ships.plugins.iam.role_ship.lambda_handler -# Runtime: python3.12 +# PackageType: Image +# ImageConfig: +# Command: +# - starfleet.worker_ships.plugins.iam.role_ship.lambda_handler # Architectures: # - arm64 # MemorySize: 128 @@ -373,10 +399,15 @@ Resources: Type: AWS::Serverless::Function DependsOn: - AccountIndexGeneratorQueue + Metadata: + DockerTag: starfleet + DockerContext: ./ + Dockerfile: Dockerfile Properties: - CodeUri: ./src - Handler: starfleet.starbase.entrypoints.fanout_payload_lambda_handler - Runtime: python3.12 + PackageType: Image + ImageConfig: + Command: + - starfleet.starbase.entrypoints.fanout_payload_lambda_handler Architectures: - arm64 Events: diff --git a/test_sam_template_NO_ECR.yaml b/test_sam_template_NO_ECR.yaml new file mode 100644 index 0000000..0ead0cf --- /dev/null +++ b/test_sam_template_NO_ECR.yaml @@ -0,0 +1,489 @@ +# This is the NON-ECR version of the SAM template +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + starfleet + + This is a Sample SAM Template for starfleet -- please replace this with your own and add your own values in. + See the Starfleet documentation here for more details: https://gemini-oss.github.io/starfleet/installation/Overview/ + +# Replace these with your own: +Parameters: + EnvironmentName: + Type: String + Default: TEST + AllowedValues: + - TEST + - PROD + Description: Informs the Lambda functions which configuration to use, either TEST or PROD + +# Environment specific configuration can go in here: +Mappings: + EnvMap: + TEST: + # Add more values that can be referenced easily here: + BaseRoleName: starfleet-worker-basic-test-role + + PROD: + # Add more values that can be referenced easily here: + BaseRoleName: starfleet-worker-basic-prod-role + +# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst +Globals: + Function: + Timeout: 300 + Environment: + Variables: + GIT_PYTHON_REFRESH: quiet # Required for IAMbic in the IAM Role Worker + +Resources: + # The Starbase Components + StarbaseFanoutDLQ: + Type: AWS::SQS::Queue + Properties: + QueueName: starbase-fanout-dlq + RedriveAllowPolicy: + redrivePermission: allowAll + + StarbaseFanoutQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: starbase-fanout-queue + VisibilityTimeout: 300 # This needs to be the same as the Lambda function timeout. + RedrivePolicy: + deadLetterTargetArn: !GetAtt StarbaseFanoutDLQ.Arn + maxReceiveCount: 4 + + StarbaseFanoutQueuePolicy: # SQS policy to allow S3 bucket notifications + Type: AWS::SQS::QueuePolicy + Properties: + Queues: + - !GetAtt StarbaseFanoutQueue.QueueUrl + PolicyDocument: + Statement: + - Effect: Allow + Action: SQS:SendMessage + Resource: + - !GetAtt StarbaseFanoutQueue.Arn + Principal: + Service: s3.amazonaws.com + Condition: + ArnLike: + "aws:SourceArn": !Sub + - 'arn:aws:s3:::starfleet-templates-${AccountId}' + - AccountId: !Ref "AWS::AccountId" + StringEquals: + "aws:SourceAccount": !Ref "AWS::AccountId" + + StarfleetTemplateBucket: + DependsOn: + - StarbaseFanoutQueuePolicy + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub + - 'starfleet-templates-${AccountId}' + - AccountId: !Ref "AWS::AccountId" + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + NotificationConfiguration: + QueueConfigurations: + - Event: "s3:ObjectCreated:*" + Queue: !GetAtt StarbaseFanoutQueue.Arn + + # The S3 bucket that holds the account index: + AccountIndexBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub + - 'starfleet-account-index-${AccountId}' + - AccountId: !Ref "AWS::AccountId" + PublicAccessBlockConfiguration: + BlockPublicAcls: True + BlockPublicPolicy: True + IgnorePublicAcls: True + RestrictPublicBuckets: True + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + + # Secrets Manager for Starfleet secrets -- this is optional and disabled by default -- recommend you click ops this actually + #StarfleetSecrets: + # Type: AWS::SecretsManager::Secret + # Properties: + # Description: Secret storage for Starfleet secrets + # KmsKeyId: alias/aws/secretsmanager + # Name: StarfleetSecrets + # Tags: + # - Add here + + # The Starbase EventBridge listener: + StarbaseEventBridgeFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: ./src + Handler: starfleet.starbase.entrypoints.eventbridge_timed_lambda_handler + Runtime: python3.12 + Architectures: + - arm64 + Events: + # Make one for each timed event: -- Don't forget to set these to Enabled when you are ready to use them! + EventBridgeFiveMin: + Type: Schedule + Properties: + Name: StarfleetFiveMin + Description: Invokes Starfleet 5-minute worker ship tasks + Enabled: False # Disabled by default for cost unless you need this for some workers + Schedule: rate(5 minutes) + Input: '{"name": "FIVE_MIN"}' + EventBridgeFifteenMin: + Type: Schedule + Properties: + Name: StarfleetFifteenMin + Description: Invokes Starfleet 15-minute worker ship tasks + Enabled: False # Disabled by default for cost unless you need this for some workers + Schedule: rate(15 minutes) + Input: '{"name": "FIFTEEN_MIN"}' + EventBridgeThirtyMin: + Type: Schedule + Properties: + Name: StarfleetThirtyMin + Description: Invokes Starfleet 30-minute worker ship tasks + Enabled: False # Disabled by default for cost unless you need this for some workers + Schedule: rate(30 minutes) + Input: '{"name": "THIRTY_MIN"}' + EventBridgeHourly: + Type: Schedule + Properties: + Name: StarfleetHourly + Description: Invokes Starfleet hourly worker ship tasks + Enabled: True + Schedule: rate(1 hour) + Input: '{"name": "HOURLY"}' + EventBridgeSixHourly: + Type: Schedule + Properties: + Name: StarfleetSixHourly + Description: Invokes Starfleet six-hourly worker ship tasks + Enabled: True + Schedule: rate(6 hours) + Input: '{"name": "SIX_HOURLY"}' + EventBridgeTwelveHourly: + Type: Schedule + Properties: + Name: StarfleetTwelveHourly + Description: Invokes Starfleet twelve-hourly worker ship tasks + Enabled: True + Schedule: rate(12 hours) + Input: '{"name": "TWELVE_HOURLY"}' + EventBridgeDaily: + Type: Schedule + Properties: + Name: StarfleetDaily + Description: Invokes Starfleet daily worker ship tasks + Enabled: True + Schedule: rate(1 day) + Input: '{"name": "DAILY"}' + Policies: + # Grant permissions to read from the Template S3 bucket and also the account index bucket: + - S3ReadPolicy: + BucketName: !Ref StarfleetTemplateBucket + - S3ReadPolicy: + BucketName: !Ref AccountIndexBucket + # Write out to the fan out queue: + - SQSSendMessagePolicy: + QueueName: !GetAtt StarbaseFanoutQueue.QueueName + + # ##### + # Define the worker ships here: + # #### + + # Account Index Generator Worker Ship Resources: + AccountIndexGeneratorDLQ: + Type: AWS::SQS::Queue + Properties: + QueueName: starfleet-account-index-generator-dlq + RedriveAllowPolicy: + redrivePermission: allowAll + + AccountIndexGeneratorQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: starfleet-account-index-generator + VisibilityTimeout: 300 # This needs to be the same as the Lambda function timeout. + RedrivePolicy: + deadLetterTargetArn: !GetAtt AccountIndexGeneratorDLQ.Arn + maxReceiveCount: 4 + + AccountIndexGenerator: + Type: AWS::Serverless::Function + Properties: + CodeUri: ./src + Handler: starfleet.worker_ships.plugins.account_index_generator.ship.lambda_handler + Runtime: python3.12 + Architectures: + - arm64 + MemorySize: 256 + Events: + SQSEvent: + Type: SQS + Properties: + Queue: !GetAtt AccountIndexGeneratorQueue.Arn + BatchSize: 2 + Environment: + Variables: + STARFLEET_COMMIT: True + Policies: + # Grant permissions to read and write to the inventory S3 bucket: + - S3ReadPolicy: + BucketName: !Ref AccountIndexBucket + - S3WritePolicy: + BucketName: !Ref AccountIndexBucket + +# Uncomment for the AWS Config worker to have that set up and deployed: +# AWSConfigWorkerDLQ: +# Type: AWS::SQS::Queue +# Properties: +# QueueName: starfleet-aws-config-worker-dlq +# RedriveAllowPolicy: +# redrivePermission: allowAll +# +# AWSConfigWorkerQueue: +# Type: AWS::SQS::Queue +# Properties: +# QueueName: starfleet-aws-config-worker +# VisibilityTimeout: 300 # This needs to be the same as the Lambda function timeout. +# RedrivePolicy: +# deadLetterTargetArn: !GetAtt AWSConfigWorkerDLQ.Arn +# maxReceiveCount: 4 +# +# AWSConfigWorker: +# Type: AWS::Serverless::Function +# Properties: +# CodeUri: ./src +# Handler: starfleet.worker_ships.plugins.aws_config.ship.lambda_handler +# Runtime: python3.12 +# Architectures: +# - arm64 +# MemorySize: 128 +# Events: +# SQSEvent: +# Type: SQS +# Properties: +# Queue: !GetAtt AWSConfigWorkerQueue.Arn +# BatchSize: 2 +# Environment: +# Variables: +# STARFLEET_COMMIT: False +# Policies: +# # Grant permissions to read from the inventory S3 bucket: +# - S3ReadPolicy: +# BucketName: !Ref AccountIndexBucket + +# Uncomment for the GitHub Repo Sync Worker: +# GitHubSyncWorkerDLQ: +# Type: AWS::SQS::Queue +# Properties: +# QueueName: starfleet-github-sync-worker-dlq +# RedriveAllowPolicy: +# redrivePermission: allowAll +# +# GitHubSyncWorkerQueue: +# Type: AWS::SQS::Queue +# Properties: +# QueueName: starfleet-github-sync-worker +# VisibilityTimeout: 300 # This needs to be the same as the Lambda function timeout. +# RedrivePolicy: +# deadLetterTargetArn: !GetAtt GitHubSyncWorkerDLQ.Arn +# maxReceiveCount: 4 +# +# GitHubSyncWorker: +# Type: AWS::Serverless::Function +# Properties: +# CodeUri: ./src +# Handler: starfleet.worker_ships.plugins.github_sync.ship.lambda_handler +# Runtime: python3.12 +# Architectures: +# - arm64 +# MemorySize: 128 +# Events: +# SQSEvent: +# Type: SQS +# Properties: +# Queue: !GetAtt GitHubSyncWorkerQueue.Arn +# BatchSize: 2 +# Environment: +# Variables: +# STARFLEET_COMMIT: False +# Policies: +# # Grant permissions to read and write to the templates bucket, so you can do CI/CD! +# - S3CrudPolicy: +# BucketName: !Ref StarfleetTemplateBucket +# # Add any additional S3 policies here! + +# Uncomment for the IAM Role Worker: +# IamRoleWorkerDLQ: +# Type: AWS::SQS::Queue +# Properties: +# QueueName: starfleet-iam-role-worker-dlq +# RedriveAllowPolicy: +# redrivePermission: allowAll +# +# IamRoleWorkerQueue: +# Type: AWS::SQS::Queue +# Properties: +# QueueName: starfleet-iam-role-worker +# VisibilityTimeout: 300 # This needs to be the same as the Lambda function timeout. +# RedrivePolicy: +# deadLetterTargetArn: !GetAtt IamRoleWorkerDLQ.Arn +# maxReceiveCount: 4 +# +# IamRoleWorker: +# Type: AWS::Serverless::Function +# Properties: +# CodeUri: ./starfleet/src +# Handler: starfleet.worker_ships.plugins.iam.role_ship.lambda_handler +# Runtime: python3.12 +# Architectures: +# - arm64 +# MemorySize: 128 +# Events: +# SQSEvent: +# Type: SQS +# Properties: +# Queue: !GetAtt IamRoleWorkerQueue.Arn +# BatchSize: 4 +# Environment: +# Variables: +# STARFLEET_COMMIT: False +# GIT_PYTHON_REFRESH: quiet # Iambic +# Policies: +# # Grant permissions to read from the Account Inventory S3 bucket: +# - S3ReadPolicy: +# BucketName: !Ref AccountIndexBucket + + # Finally, the Starbase Fanout Function: + StarbaseFanoutFunction: + Type: AWS::Serverless::Function + DependsOn: + - AccountIndexGeneratorQueue + Properties: + CodeUri: ./src + Handler: starfleet.starbase.entrypoints.fanout_payload_lambda_handler + Runtime: python3.12 + Architectures: + - arm64 + Events: + # Make one for each timed event: + SQSEvent: + Type: SQS + Properties: + Queue: !GetAtt StarbaseFanoutQueue.Arn + BatchSize: 1 # Important! We only want 1 to ensure we don't hit timeouts. + Policies: + # Grant permissions to read from the Template S3 bucket: + - S3ReadPolicy: + BucketName: !Ref StarfleetTemplateBucket + - SQSSendMessagePolicy: + QueueName: starfleet-* + + # Common worker ship components here: + AssumeRoleManagedPolicy: + Type: AWS::IAM::ManagedPolicy + DependsOn: + - AccountIndexGenerator + Properties: + Description: Grants Starfleet workers assume role permissions to common Starfleet worker IAM roles + ManagedPolicyName: StarfleetWorkerAssumeRoles + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: 'sts:AssumeRole' + Resource: + - !Sub + - 'arn:aws:iam::*:role/${RoleName}' + - RoleName: !FindInMap + - EnvMap + - !Ref 'EnvironmentName' + - BaseRoleName + # Uncomment for secrets manager permissions as well for Slack alerts and GitHub Repo Sync worker + # - Effect: Allow + # Action: secretsmanager:GetSecretValue + # Resource: arn:aws:secretsmanager:REGION:ACCOUNTNUMBER:secret:STARFLEET-SECRET-ID-HERE + + Roles: + - !Ref AccountIndexGeneratorRole # AccountIndexGeneratorRole is created automatically by SAM and can be referenced + # - !Ref AWSConfigWorkerRole # Uncomment for the AWS Config worker + # - !Ref GitHubSyncWorkerRole # Uncomment for the GitHub Repo Sync Worker + # - !Ref IamRoleWorkerRole # Uncomment for the IAM Role Worker + # - !Ref YourWorkerRoleHere # Add your worker roles here to get the policy attached + +Outputs: + StarbaseFanoutQueue: + Description: The Queue URL for the Starbase fan out queue + Value: !GetAtt StarbaseFanoutQueue.QueueUrl + StarbaseFanoutDLQ: + Description: The Queue URL for the Starbase fan out DLQ + Value: !GetAtt StarbaseFanoutDLQ.QueueUrl + StarfleetTemplateBucket: + Description: The Starfleet S3 bucket that holds all the worker ship templates + Value: !Ref StarfleetTemplateBucket + StarbaseEventBridgeFunction: + Description: The primary entrypoint for the Starbase timed EventBridge invocations + Value: !GetAtt StarbaseEventBridgeFunction.Arn + StarbaseFanoutFunction: + Description: The Starbase function that fans out the tasks to the worker ships + Value: !GetAtt StarbaseFanoutFunction.Arn + AssumeRoleManagedPolicy: + Description: The Assume Role managed policy that can be attached to worker roles that need it + Value: !Ref AssumeRoleManagedPolicy + AccountIndexBucket: + Description: The Starfleet S3 bucket that holds the account index JSON file + Value: !Ref AccountIndexBucket + AccountIndexGeneratorQueue: + Description: The Queue URL for the Account Index Generator invocation queue + Value: !GetAtt AccountIndexGeneratorQueue.QueueUrl + AccountIndexGeneratorDLQ: + Description: The Queue URL for the Account Index Generator invocation DLQ + Value: !GetAtt AccountIndexGeneratorDLQ.QueueUrl + AccountIndexGeneratorFunction: + Description: The Account Index Generator worker ship function that generates an account index + Value: !GetAtt AccountIndexGenerator.Arn +# Uncomment for the AWS Config worker: +# AWSConfigWorkerQueue: +# Description: The Queue URL for the AWS Config Worker invocation queue +# Value: !GetAtt AWSConfigWorkerQueue.QueueUrl +# AWSConfigWorkerDLQ: +# Description: The Queue URL for the AWS Config Worker invocation DLQ +# Value: !GetAtt AWSConfigWorkerDLQ.QueueUrl +# AWSConfigWorker: +# Description: The AWS Config Worker ship function that enables AWS Config everywhere +# Value: !GetAtt AWSConfigWorker.Arn +# Uncomment for the GitHub Sync worker: +# GitHubSyncWorkerQueue: +# Description: The Queue URL for the GitHub Sync Worker invocation queue +# Value: !GetAtt GitHubSyncWorkerQueue.QueueUrl +# GitHubSyncWorkerDLQ: +# Description: The Queue URL for the GitHub Sync Worker invocation DLQ +# Value: !GetAtt GitHubSyncWorkerDLQ.QueueUrl +# GitHubSyncWorker: +# Description: The GitHub Sync Worker ship function that syncs a GitHub repo with S3 +# Value: !GetAtt GitHubSyncWorker.Arn +# Uncomment for the IAM Role worker: +# IamRoleWorkerQueue: +# Description: The Queue URL for the IAM Role Worker invocation queue +# Value: !GetAtt IamRoleWorkerQueue.QueueUrl +# IamRoleWorkerDLQ: +# Description: The Queue URL for the IAM Role Worker invocation DLQ +# Value: !GetAtt IamRoleWorkerDLQ.QueueUrl +# IamRoleWorker: +# Description: The IAM Role Worker ship function that uses iambic to sync IAM roles in our infrastructure +# Value: !GetAtt IamRoleWorker.Arn