diff --git a/.gitignore b/.gitignore index 0807788..c718e64 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,6 @@ # Output from other tools that might be used alongside Terraform/OpenTofu *.tfvars.json -backend.tf.json # Taskit files .taskit/ @@ -39,4 +38,4 @@ backend.tf.json **/*.temp **/*.bak **/*.*swp -**/.DS_Store \ No newline at end of file +**/.DS_Store diff --git a/README.md b/README.md index 865dbf2..df17afc 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,194 @@ -# terraform-module-template +# `spacelift-automation` -[![Release](https://img.shields.io/github/release/masterpointio/terraform-module-template.svg)](https://github.com/masterpointio/terraform-module-template/releases/latest) +[![Release](https://img.shields.io/github/release/masterpointio/terraform-spacelift-automation.svg)](https://github.com/masterpointio/terraform-spacelift-automation/releases/latest) -This repository serves as a template for creating Terraform modules, providing a standardized structure and essential files for efficient module development. It's designed to ensure consistency and best practices across Terraform projects. +This Terraform child module provides infrastructure automation for projects in [Spacelift](https://docs.spacelift.io/). + +## Overview + +The `spacelift-automation` root module is designed to streamline the deployment and management of all Spacelift infrastructure, including itself. + +It automates the creation of "child" stacks and all the required accompanying Spacelift resources. For each enabled root module it creates: + +1. [Spacelift Stack](https://docs.spacelift.io/concepts/stack/) + You can think about a stack as a combination of source code, state file and configuration in the form of environment variables and mounted files. +2. [Spacelift Stack Destructor](https://docs.spacelift.io/concepts/stack/stack-dependencies.html#ordered-stack-creation-and-deletion) + Required to destroy the resources of a Stack before deleting it. Destroying this resource will delete the resources in the stack. If this resource needs to be deleted and the resources in the stacks are to be preserved, ensure that the deactivated attribute is set to true. +3. [Spacelift AWS Integration Attachment](https://docs.spacelift.io/integrations/cloud-providers/aws#lets-explain) + Associates a specific AWS IAM role with a stack to allow it to assume that role. The IAM role typically has permissions to manage specific AWS resources, and Spacelift assumes this role to run the operations required by the stack. +4. [Spacelift Initialization Hook](https://docs.spacelift.io/concepts/run#initializing) + Prepares your environment before executing infrastructure code. This custom script copies corresponding Terraform tfvars files into a working directory before either run or task as a `spacelift.auto.tfvars` file. It's [automatically loaded](https://opentofu.org/docs/v1.7/language/values/variables/#variable-definitions-tfvars-files) into the OpenTofu/Terraform execution environment. ## Usage -TODO +Spacelift Automation logic is opinionated and heavily relies on the Git repository structure. +This module is configured to track all the files in the provided root module directory and create the stack based on the provided configuration (if any). + +Structure requirements are: + +- Stack configs are placed in `/stacks` directory. +- Terraform variables are placed in `/tfvars` directory. +- Stack config file and tfvars file must be equal to OpenTofu/Terraform workspace, e.g. `dev.yaml` and `dev.tfvars`. +- Common configs are placed in `/stacks/common.yaml` file. This is useful when you know that some values should be shared across all the stacks created for a root module, e.g. all stacks that manage Spacelift Policy must be Administrative. You can override this file name using Terraform variable. + +Let's check the example. +Input repo structure: + +```sh +├── root-modules +│ ├── spacelift-aws-role +│ │ ├── stacks +│ │ │ └── dev.yaml +│ │ │ └── stage.yaml +│ │ │ └── common.yaml +│ │ ├── tfvars +│ │ │ └── dev.tfvars +│ │ │ └── stage.tfvars +│ │ ├── variables.tf +│ │ └── versions.tf +│ ├── k8s-cluster +│ │ ├── stacks +│ │ │ └── dev.yaml +│ │ │ └── prod.yaml +│ │ │ └── common.yaml +│ │ ├── tfvars +│ │ │ └── dev.tfvars +│ │ │ └── prod.tfvars +│ │ ├── variables.tf +│ │ ├── main.tf +│ │ └── versions.tf +... +``` + +Root module inputs: + +```hcl +aws_integration_id = "ZDPP8SKNVG0G27T4" + +# GitHub configuration +github_enterprise = { + namespace = "masterpointio" +} +repository = "terraform-spacelift-automation" + +# Stacks configurations +root_modules_path = "root-modules" +enabled_root_modules = ["spacelift-aws-role"] +``` + +The configuration above creates the following stacks: + +- `spacelift-aws-role-dev` +- `spacelift-aws-role-stage` + +These stacks have the following configuration: + +- Stacks track changes in GitHub repo `github.com/masterpointio/terraform-spacelift-automation`, branch `main` (the default), and directrory `root-modules`. +- Common configuration is defined in `root-modules/spacelift-aws-role/stacks/common.yaml` and applied to both Stacks. However, if there is an override in a Stack config (e.g. `root-modules/spacelift-aws-role/stacks/dev.yaml`), it takes precedence over common configs. +- Corresponding Terraform variables are generated by an [Initialization Hook](https://docs.spacelift.io/concepts/run#initializing) and placed in the root of each Stack's working directory during each run or task. For example, the content of the file `root-modules/spacelift-aws-role/tfvars/dev.tfvars` will be copied to working directory of the Stack `spacelift-aws-role-dev` as file `spacelift.auto.tfvars` allowing the OpenTofu/Terraform inputs to be automatically loaded. + - If you would like to disable this functionality, you can set `tfvars.enabled` in the Stack's YAML file to `false`. + +## FAQs + +### Why are variable values provided separately in `tfvars/` and not in the `yaml` file? + +This is to support easy local and outside-spacelift operations. Keeping variable values in a `tfvars` file per workspace allows you to simply pass that file to the relevant CLI command locally via the `-var-file` option so that you don't need to provide values individually. + +### Can I create a Spacelift Stack for Spacelift Automation? (Recommended) + +Spacelift Automation can manage itself as a Stack as well, and we recommend this so you can fully automate your Stack management upon merging to your given branch. Follow these next steps to achieve that: + +1. Create a new vanilla OpenTofu/Terraform root module that consumes this child module and supplies the necessary configuration for your unique setup. In other words, it's a configuration that uses the default capabilities of either OpenTofu or Terraform without any customization, or third-party tools or plugins. +2. Optionally, create a Terraform workspace that will be used for your Automation configuration, e.g.: + ```sh + tofu workspace new masterpoint + ``` + Remember that Stack config and tfvars file name must be equal to the workspace, which can be `default`. +3. Apply the vanilla OpenTofu/Terraform configuration. +4. Move the Automation configs to the `/spacelift-automation/stacks` directory and push the changes to the tracked repo and branch. +5. From this moment, Spacelift Automation is tracking the changes to its Stack configs and Terraform variables. + +Check out an example of such a configuration in the [examples/complete](./examples/complete/components/spacelift-automation/tfvars/example.tfvars). + +NOTE to Masterpoint team: We might want to create a small wrapper to automatize this using Taskit. On hold for now. ## Requirements -No requirements. +| Name | Version | +| ------------------------------------------------------------------------ | ------- | +| [terraform](#requirement_terraform) | >= 1.6 | +| [spacelift](#requirement_spacelift) | >= 1.14 | ## Providers -No provider. +| Name | Version | +| ------------------------------------------------------------------ | ------- | +| [spacelift](#provider_spacelift) | 1.16.1 | + +## Modules + +| Name | Source | Version | +| ----------------------------------------------- | ----------------------------------------- | ------- | +| [deep](#module_deep) | cloudposse/config/yaml//modules/deepmerge | 1.0.2 | + +## Resources + +| Name | Type | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| [spacelift_aws_integration_attachment.default](https://registry.terraform.io/providers/spacelift-io/spacelift/latest/docs/resources/aws_integration_attachment) | resource | +| [spacelift_drift_detection.default](https://registry.terraform.io/providers/spacelift-io/spacelift/latest/docs/resources/drift_detection) | resource | +| [spacelift_stack.default](https://registry.terraform.io/providers/spacelift-io/spacelift/latest/docs/resources/stack) | resource | +| [spacelift_stack_destructor.default](https://registry.terraform.io/providers/spacelift-io/spacelift/latest/docs/resources/stack_destructor) | resource | ## Inputs -No input. +| Name | Description | Type | Default | Required | +| --------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | ---------------------------------------------------------- | :------: | +| [administrative](#input_administrative) | Flag to mark the stack as administrative | `bool` | `false` | no | +| [after_apply](#input_after_apply) | List of after-apply scripts | `list(string)` | `[]` | no | +| [after_destroy](#input_after_destroy) | List of after-destroy scripts | `list(string)` | `[]` | no | +| [after_init](#input_after_init) | List of after-init scripts | `list(string)` | `[]` | no | +| [after_perform](#input_after_perform) | List of after-perform scripts | `list(string)` | `[]` | no | +| [after_plan](#input_after_plan) | List of after-plan scripts | `list(string)` | `[]` | no | +| [all_root_modules_enabled](#input_all_root_modules_enabled) | When set to true, all subdirectories in root_modules_path will be treated as root modules. | `bool` | `false` | no | +| [autodeploy](#input_autodeploy) | Flag to enable/disable automatic deployment of the stack | `bool` | `true` | no | +| [autoretry](#input_autoretry) | Flag to enable/disable automatic retry of the stack | `bool` | `false` | no | +| [aws_integration_attachment_read](#input_aws_integration_attachment_read) | Indicates whether this attachment is used for read operations. | `bool` | `true` | no | +| [aws_integration_attachment_write](#input_aws_integration_attachment_write) | Indicates whether this attachment is used for write operations. | `bool` | `true` | no | +| [aws_integration_enabled](#input_aws_integration_enabled) | Indicates whether the AWS integration is enabled. | `bool` | `false` | no | +| [aws_integration_id](#input_aws_integration_id) | ID of the AWS integration to attach. | `string` | `null` | no | +| [before_apply](#input_before_apply) | List of before-apply scripts | `list(string)` | `[]` | no | +| [before_destroy](#input_before_destroy) | List of before-destroy scripts | `list(string)` | `[]` | no | +| [before_init](#input_before_init) | List of before-init scripts | `list(string)` | `[]` | no | +| [before_perform](#input_before_perform) | List of before-perform scripts | `list(string)` | `[]` | no | +| [before_plan](#input_before_plan) | List of before-plan scripts | `list(string)` | `[]` | no | +| [branch](#input_branch) | Specify which branch to use within the infrastructure repository. | `string` | `"main"` | no | +| [common_config_file](#input_common_config_file) | Name of the common configuration file for the stack across a root module. | `string` | `"common.yaml"` | no | +| [description](#input_description) | Description of the stack | `string` | `"Managed by spacelift-automation Terraform root module."` | no | +| [destructor_enabled](#input_destructor_enabled) | Flag to enable/disable the destructor for the Stack. | `bool` | `false` | no | +| [drift_detection_enabled](#input_drift_detection_enabled) | Flag to enable/disable Drift Detection configuration for a Stack. | `bool` | `false` | no | +| [drift_detection_ignore_state](#input_drift_detection_ignore_state) | Controls whether drift detection should be performed on a stack
in any final state instead of just 'Finished'. | `bool` | `false` | no | +| [drift_detection_reconcile](#input_drift_detection_reconcile) | Flag to enable/disable automatic reconciliation of drifts. | `bool` | `false` | no | +| [drift_detection_schedule](#input_drift_detection_schedule) | The schedule for drift detection. | `list(string)` | 
[
 "0 4 * * *"
]
| no | +| [drift_detection_timezone](#input_drift_detection_timezone) | The timezone for drift detection. | `string` | `"UTC"` | no | +| [enable_local_preview](#input_enable_local_preview) | Indicates whether local preview runs can be triggered on this Stack. | `bool` | `false` | no | +| [enabled_root_modules](#input_enabled_root_modules) | List of root modules where to look for stack config files.
Ignored when all_root_modules_enabled is true.
Example: ["spacelift-automation", "k8s-cluster"] | `list(string)` | `[]` | no | +| [github_enterprise](#input_github_enterprise) | The GitHub VCS settings | 
object({
 namespace = string
 id = optional(string)
 })
| n/a | yes | +| [manage_state](#input_manage_state) | Determines if Spacelift should manage state for this stack. | `bool` | `false` | no | +| [protect_from_deletion](#input_protect_from_deletion) | Protect this stack from accidental deletion. If set, attempts to delete this stack will fail. | `bool` | `false` | no | +| [repository](#input_repository) | The name of your infrastructure repo | `string` | n/a | yes | +| [root_modules_path](#input_root_modules_path) | The path, relative to the root of the repository, where the root module can be found. | `string` | `"root-modules"` | no | +| [space_id](#input_space_id) | Place the stack in the specified space_id. | `string` | `"root"` | no | +| [terraform_smart_sanitization](#input_terraform_smart_sanitization) | Indicates whether runs on this will use terraform's sensitive value system to sanitize
the outputs of Terraform state and plans in spacelift instead of sanitizing all fields. | `bool` | `false` | no | +| [terraform_version](#input_terraform_version) | Terraform version to use. | `string` | `"1.7.2"` | no | +| [terraform_workflow_tool](#input_terraform_workflow_tool) | Defines the tool that will be used to execute the workflow.
This can be one of OPEN_TOFU, TERRAFORM_FOSS or CUSTOM. | `string` | `"OPEN_TOFU"` | no | +| [worker_pool_id](#input_worker_pool_id) | ID of the worker pool to use.
NOTE: worker_pool_id is required when using a self-hosted instance of Spacelift. | `string` | `null` | no | ## Outputs -No output. +No outputs. @@ -32,7 +196,7 @@ No output. Contributions are welcome and appreciated! -Found an issue or want to request a feature? [Open an issue](TODO) +Found an issue or want to request a feature? [Open an issue](https://github.com/masterpointio/terraform-spacelift-automation/issues/new) Want to fix a bug you found or add some functionality? Fork, clone, commit, push, and PR and we'll check it out. diff --git a/context.tf b/context.tf deleted file mode 100644 index 5e0ef88..0000000 --- a/context.tf +++ /dev/null @@ -1,279 +0,0 @@ -# -# ONLY EDIT THIS FILE IN github.com/cloudposse/terraform-null-label -# All other instances of this file should be a copy of that one -# -# -# Copy this file from https://github.com/cloudposse/terraform-null-label/blob/master/exports/context.tf -# and then place it in your Terraform module to automatically get -# Cloud Posse's standard configuration inputs suitable for passing -# to Cloud Posse modules. -# -# curl -sL https://raw.githubusercontent.com/cloudposse/terraform-null-label/master/exports/context.tf -o context.tf -# -# Modules should access the whole context as `module.this.context` -# to get the input variables with nulls for defaults, -# for example `context = module.this.context`, -# and access individual variables as `module.this.`, -# with final values filled in. -# -# For example, when using defaults, `module.this.context.delimiter` -# will be null, and `module.this.delimiter` will be `-` (hyphen). -# - -module "this" { - source = "cloudposse/label/null" - version = "0.25.0" # requires Terraform >= 0.13.0 - - enabled = var.enabled - namespace = var.namespace - tenant = var.tenant - environment = var.environment - stage = var.stage - name = var.name - delimiter = var.delimiter - attributes = var.attributes - tags = var.tags - additional_tag_map = var.additional_tag_map - label_order = var.label_order - regex_replace_chars = var.regex_replace_chars - id_length_limit = var.id_length_limit - label_key_case = var.label_key_case - label_value_case = var.label_value_case - descriptor_formats = var.descriptor_formats - labels_as_tags = var.labels_as_tags - - context = var.context -} - -# Copy contents of cloudposse/terraform-null-label/variables.tf here - -variable "context" { - type = any - default = { - enabled = true - namespace = null - tenant = null - environment = null - stage = null - name = null - delimiter = null - attributes = [] - tags = {} - additional_tag_map = {} - regex_replace_chars = null - label_order = [] - id_length_limit = null - label_key_case = null - label_value_case = null - descriptor_formats = {} - # Note: we have to use [] instead of null for unset lists due to - # https://github.com/hashicorp/terraform/issues/28137 - # which was not fixed until Terraform 1.0.0, - # but we want the default to be all the labels in `label_order` - # and we want users to be able to prevent all tag generation - # by setting `labels_as_tags` to `[]`, so we need - # a different sentinel to indicate "default" - labels_as_tags = ["unset"] - } - description = <<-EOT - Single object for setting entire context at once. - See description of individual variables for details. - Leave string and numeric variables as `null` to use default value. - Individual variable settings (non-null) override settings in context object, - except for attributes, tags, and additional_tag_map, which are merged. - EOT - - validation { - condition = lookup(var.context, "label_key_case", null) == null ? true : contains(["lower", "title", "upper"], var.context["label_key_case"]) - error_message = "Allowed values: `lower`, `title`, `upper`." - } - - validation { - condition = lookup(var.context, "label_value_case", null) == null ? true : contains(["lower", "title", "upper", "none"], var.context["label_value_case"]) - error_message = "Allowed values: `lower`, `title`, `upper`, `none`." - } -} - -variable "enabled" { - type = bool - default = null - description = "Set to false to prevent the module from creating any resources" -} - -variable "namespace" { - type = string - default = null - description = "ID element. Usually an abbreviation of your organization name, e.g. 'eg' or 'cp', to help ensure generated IDs are globally unique" -} - -variable "tenant" { - type = string - default = null - description = "ID element _(Rarely used, not included by default)_. A customer identifier, indicating who this instance of a resource is for" -} - -variable "environment" { - type = string - default = null - description = "ID element. Usually used for region e.g. 'uw2', 'us-west-2', OR role 'prod', 'staging', 'dev', 'UAT'" -} - -variable "stage" { - type = string - default = null - description = "ID element. Usually used to indicate role, e.g. 'prod', 'staging', 'source', 'build', 'test', 'deploy', 'release'" -} - -variable "name" { - type = string - default = null - description = <<-EOT - ID element. Usually the component or solution name, e.g. 'app' or 'jenkins'. - This is the only ID element not also included as a `tag`. - The "name" tag is set to the full `id` string. There is no tag with the value of the `name` input. - EOT -} - -variable "delimiter" { - type = string - default = null - description = <<-EOT - Delimiter to be used between ID elements. - Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. - EOT -} - -variable "attributes" { - type = list(string) - default = [] - description = <<-EOT - ID element. Additional attributes (e.g. `workers` or `cluster`) to add to `id`, - in the order they appear in the list. New attributes are appended to the - end of the list. The elements of the list are joined by the `delimiter` - and treated as a single ID element. - EOT -} - -variable "labels_as_tags" { - type = set(string) - default = ["default"] - description = <<-EOT - Set of labels (ID elements) to include as tags in the `tags` output. - Default is to include all labels. - Tags with empty values will not be included in the `tags` output. - Set to `[]` to suppress all generated tags. - **Notes:** - The value of the `name` tag, if included, will be the `id`, not the `name`. - Unlike other `null-label` inputs, the initial setting of `labels_as_tags` cannot be - changed in later chained modules. Attempts to change it will be silently ignored. - EOT -} - -variable "tags" { - type = map(string) - default = {} - description = <<-EOT - Additional tags (e.g. `{'BusinessUnit': 'XYZ'}`). - Neither the tag keys nor the tag values will be modified by this module. - EOT -} - -variable "additional_tag_map" { - type = map(string) - default = {} - description = <<-EOT - Additional key-value pairs to add to each map in `tags_as_list_of_maps`. Not added to `tags` or `id`. - This is for some rare cases where resources want additional configuration of tags - and therefore take a list of maps with tag key, value, and additional configuration. - EOT -} - -variable "label_order" { - type = list(string) - default = null - description = <<-EOT - The order in which the labels (ID elements) appear in the `id`. - Defaults to ["namespace", "environment", "stage", "name", "attributes"]. - You can omit any of the 6 labels ("tenant" is the 6th), but at least one must be present. - EOT -} - -variable "regex_replace_chars" { - type = string - default = null - description = <<-EOT - Terraform regular expression (regex) string. - Characters matching the regex will be removed from the ID elements. - If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. - EOT -} - -variable "id_length_limit" { - type = number - default = null - description = <<-EOT - Limit `id` to this many characters (minimum 6). - Set to `0` for unlimited length. - Set to `null` for keep the existing setting, which defaults to `0`. - Does not affect `id_full`. - EOT - validation { - condition = var.id_length_limit == null ? true : var.id_length_limit >= 6 || var.id_length_limit == 0 - error_message = "The id_length_limit must be >= 6 if supplied (not null), or 0 for unlimited length." - } -} - -variable "label_key_case" { - type = string - default = null - description = <<-EOT - Controls the letter case of the `tags` keys (label names) for tags generated by this module. - Does not affect keys of tags passed in via the `tags` input. - Possible values: `lower`, `title`, `upper`. - Default value: `title`. - EOT - - validation { - condition = var.label_key_case == null ? true : contains(["lower", "title", "upper"], var.label_key_case) - error_message = "Allowed values: `lower`, `title`, `upper`." - } -} - -variable "label_value_case" { - type = string - default = null - description = <<-EOT - Controls the letter case of ID elements (labels) as included in `id`, - set as tag values, and output by this module individually. - Does not affect values of tags passed in via the `tags` input. - Possible values: `lower`, `title`, `upper` and `none` (no transformation). - Set this to `title` and set `delimiter` to `""` to yield Pascal Case IDs. - Default value: `lower`. - EOT - - validation { - condition = var.label_value_case == null ? true : contains(["lower", "title", "upper", "none"], var.label_value_case) - error_message = "Allowed values: `lower`, `title`, `upper`, `none`." - } -} - -variable "descriptor_formats" { - type = any - default = {} - description = <<-EOT - Describe additional descriptors to be output in the `descriptors` output map. - Map of maps. Keys are names of descriptors. Values are maps of the form - `{ - format = string - labels = list(string) - }` - (Type is `any` so the map values can later be enhanced to provide additional options.) - `format` is a Terraform format string to be passed to the `format()` function. - `labels` is a list of labels, in order, to pass to `format()` function. - Label values will be normalized before being passed to `format()` so they will be - identical to how they appear in `id`. - Default is `{}` (`descriptors` output will be empty). - EOT -} - -#### End of copy of cloudposse/terraform-null-label/variables.tf diff --git a/examples/complete/README.md b/examples/complete/README.md new file mode 100644 index 0000000..e709e98 --- /dev/null +++ b/examples/complete/README.md @@ -0,0 +1,38 @@ +# Complete Example + +This example demonstrates how to use the spacelift-automation component to manage Spacelift stacks, including the ability for Spacelift to manage its own configuration. + +Normally, this directory would contain a simple root module to spin up a basic example. To showcase a more practical use case where Spacelift manages its own infrastructure, we’ve moved the configuration to the expected path: [examples/complete/components/spacelift-automation/](./components/spacelift-automation/). + +By doing this, we provide an example of how to set up Spacelift to automate the management of your infrastructure stacks, including itself. + +## Use the Example + +1. Prerequisites: + - Replace the following configuration files with your own values: + - `backend.tf.json`: Configure your Terraform backend settings + - `example.tfvars`: Set your Spacelift configuration variables + - `example.yaml`: Define your stack configuration + > **Important:** These files may contain sensitive information. Ensure you: + > + > - Remove any hardcoded credentials or sensitive values + > - Have appropriate Spacelift and AWS permissions + > - Follow your organization's security practices +1. Navigate to the spacelift-automation component directory: + ```sh + cd ./components/spacelift-automation/ + ``` +1. Initialize Terraform: + ```sh + tofu init + ``` +1. Select the worspace: + ```sh + tofu workspace select example + ``` +1. Review the Terraform plan: + ```sh + tofu plan -var-file tfvars/example.tfvars + ``` + +This will set up the Spacelift stack that manages itself. diff --git a/examples/complete/components/random-pet/main.tf b/examples/complete/components/random-pet/main.tf new file mode 100644 index 0000000..74e76fd --- /dev/null +++ b/examples/complete/components/random-pet/main.tf @@ -0,0 +1,3 @@ +resource "random_pet" "template" { + length = var.length +} diff --git a/examples/complete/components/random-pet/stacks/common.yaml b/examples/complete/components/random-pet/stacks/common.yaml new file mode 100644 index 0000000..5cdd375 --- /dev/null +++ b/examples/complete/components/random-pet/stacks/common.yaml @@ -0,0 +1,3 @@ +stack_settings: + manage_state: true + description: This stack generates random pet names diff --git a/examples/complete/components/random-pet/stacks/example.yaml b/examples/complete/components/random-pet/stacks/example.yaml new file mode 100644 index 0000000..39391c6 --- /dev/null +++ b/examples/complete/components/random-pet/stacks/example.yaml @@ -0,0 +1,2 @@ +stack_settings: + manage_state: true diff --git a/examples/complete/components/random-pet/tfvars/example.tfvars b/examples/complete/components/random-pet/tfvars/example.tfvars new file mode 100644 index 0000000..983ef02 --- /dev/null +++ b/examples/complete/components/random-pet/tfvars/example.tfvars @@ -0,0 +1 @@ +length = 10 diff --git a/examples/complete/components/random-pet/variables.tf b/examples/complete/components/random-pet/variables.tf new file mode 100644 index 0000000..e44654e --- /dev/null +++ b/examples/complete/components/random-pet/variables.tf @@ -0,0 +1,4 @@ +variable "length" { + description = "The length of the random name" + type = number +} diff --git a/examples/complete/components/random-pet/versions.tf b/examples/complete/components/random-pet/versions.tf new file mode 100644 index 0000000..e6f2511 --- /dev/null +++ b/examples/complete/components/random-pet/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "~> 1.0" + + required_providers { + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/examples/complete/components/spacelift-automation/backend.tf.json b/examples/complete/components/spacelift-automation/backend.tf.json new file mode 100644 index 0000000..e4e1f6b --- /dev/null +++ b/examples/complete/components/spacelift-automation/backend.tf.json @@ -0,0 +1,18 @@ +{ + "terraform": { + "backend": { + "s3": { + "workspace_key_prefix": "terraform-spacelift-automation", + "acl": "bucket-owner-full-control", + "bucket": "mp-automation-tfstate", + "dynamodb_table": "mp-automation-tfstate-lock", + "encrypt": true, + "key": "terraform.tfstate", + "region": "us-east-1", + "assume_role": { + "role_arn": "arn:aws:iam::755965222190:role/mp-automation-tfstate" + } + } + } + } +} diff --git a/examples/complete/components/spacelift-automation/main.tf b/examples/complete/components/spacelift-automation/main.tf new file mode 100644 index 0000000..1ca0be6 --- /dev/null +++ b/examples/complete/components/spacelift-automation/main.tf @@ -0,0 +1,12 @@ +module "automation" { + source = "../../../../" + + github_enterprise = var.github_enterprise + repository = var.repository + branch = var.branch + + root_modules_path = var.root_modules_path + all_root_modules_enabled = var.all_root_modules_enabled + + aws_integration_id = var.aws_integration_id +} diff --git a/examples/complete/components/spacelift-automation/stacks/common.yaml b/examples/complete/components/spacelift-automation/stacks/common.yaml new file mode 100644 index 0000000..69b675a --- /dev/null +++ b/examples/complete/components/spacelift-automation/stacks/common.yaml @@ -0,0 +1,3 @@ +stack_settings: + administrative: true + aws_integration_enabled: true diff --git a/examples/complete/components/spacelift-automation/stacks/example.yaml b/examples/complete/components/spacelift-automation/stacks/example.yaml new file mode 100644 index 0000000..6256f9e --- /dev/null +++ b/examples/complete/components/spacelift-automation/stacks/example.yaml @@ -0,0 +1,2 @@ +stack_settings: + description: This Automation stack is used for Masterpoint's testing purposes diff --git a/examples/complete/components/spacelift-automation/tfvars/example.tfvars b/examples/complete/components/spacelift-automation/tfvars/example.tfvars new file mode 100644 index 0000000..e7dd728 --- /dev/null +++ b/examples/complete/components/spacelift-automation/tfvars/example.tfvars @@ -0,0 +1,8 @@ +github_enterprise = { + namespace = "masterpointio" +} +repository = "terraform-spacelift-automation" +branch = "feature/initial-version" # TODO: remove this +root_modules_path = "../../../../examples/complete/components" +all_root_modules_enabled = true +aws_integration_id = "01J30JBKQTCD72ATZCRWHYST3C" diff --git a/examples/complete/components/spacelift-automation/variables.tf b/examples/complete/components/spacelift-automation/variables.tf new file mode 100644 index 0000000..395f434 --- /dev/null +++ b/examples/complete/components/spacelift-automation/variables.tf @@ -0,0 +1,35 @@ +variable "aws_integration_id" { + type = string + description = "ID of the AWS integration to attach." +} + +variable "branch" { + type = string + description = "Specify which branch to use within the infrastructure repository." + default = "main" +} + +variable "all_root_modules_enabled" { + type = bool + description = "When set to true, all subdirectories in root_modules_path will be treated as root modules." + default = false +} + +variable "github_enterprise" { + type = object({ + namespace = string + id = optional(string) + }) + description = "The GitHub VCS settings" +} + +variable "repository" { + type = string + description = "The name of your infrastructure repo" +} + +variable "root_modules_path" { + type = string + description = "The path, relative to the root of the repository, where the root module can be found." + default = "root-modules" +} diff --git a/examples/complete/components/spacelift-automation/versions.tf b/examples/complete/components/spacelift-automation/versions.tf new file mode 100644 index 0000000..a59e42a --- /dev/null +++ b/examples/complete/components/spacelift-automation/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "~> 1.0" + + required_providers { + spacelift = { + source = "spacelift-io/spacelift" + version = "~> 1.14" + } + } +} diff --git a/examples/complete/main.tf b/examples/complete/main.tf deleted file mode 100644 index f9d23f1..0000000 --- a/examples/complete/main.tf +++ /dev/null @@ -1 +0,0 @@ -# complete.tf diff --git a/examples/complete/outputs.tf b/examples/complete/outputs.tf deleted file mode 100644 index f9d23f1..0000000 --- a/examples/complete/outputs.tf +++ /dev/null @@ -1 +0,0 @@ -# complete.tf diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf deleted file mode 100644 index f9d23f1..0000000 --- a/examples/complete/variables.tf +++ /dev/null @@ -1 +0,0 @@ -# complete.tf diff --git a/main.tf b/main.tf index 74e76fd..1da3aa7 100644 --- a/main.tf +++ b/main.tf @@ -1,3 +1,322 @@ -resource "random_pet" "template" { - length = var.length +# This Terraform code automates the creation and management of Spacelift stacks based on the structure +# and configurations defined in the Git repository, default Stack values and additional input variables. +# It primarily relies on dynamic local expressions to generate configurations based on the +# input variables and Git structure so it can be consumed by Spacelift resources. +# This module can also manage the automation stack itself, but it should be bootstrapped manually. +# +# It handles the following: +# +# 1. Stack Configurations (see ## Stack Configurations) +# Reads the Spacelift stack configurations strictly based on the root modules structure in Git and file names. +# These are the configurations required to be set for a stack, e.g. project_root, terraform_workspace, root_module. +# +# 2. Common Stack configurations (see ## Common Stack configurations) +# Some configurations are equal across the whole root module, and can be set it on a root module level: +# * Space IDs: in the majority of cases all the workspaces in a root module belong to the same Spacelift space, so +# we allow setting a "global" space_id for all stacks on a root module level. +# * Autodeploy: if all the stacks in a root module should be autodeployed. +# * Administrative: if all the stacks in a root module are administrative, e.g stacks that manage Spacelift resources. +# +# 3. Labels (see ## Labels) +# Generates labels for the stacks based on administrative, dependency, and folder information. +# +# Syntax note: +# The local expression started with an underscore `_` is used to store intermediate values +# that are not directly used in the resource creation. + +locals { + _all_stack_files = fileset("${path.root}/${var.root_modules_path}/*/stacks", "*.yaml") + _all_root_modules = distinct([for file in local._all_stack_files : dirname(replace(replace(file, "../", ""), "stacks/", ""))]) + enabled_root_modules = var.all_root_modules_enabled ? local._all_root_modules : var.enabled_root_modules + + # Read and decode Stack YAML files from the root directory + # Example: + # { + # "random-pet" = { + # "common.yaml" = { + # "stack_settings" = { + # "description" = "This stack generates random pet names" + # "manage_state" = true + # } + # "tfvars" = { + # "enabled" = false + # } + # } + # "example.yaml" = { + # "stack_settings" = { + # "manage_state" = true + # } + # "tfvars" = { + # "enabled" = true + # } + # } + # } + # } + + _root_module_yaml_decoded = { + for module in local.enabled_root_modules : module => { + for yaml_file in fileset("${path.root}/${var.root_modules_path}/${module}/stacks", "*.yaml") : + yaml_file => yamldecode(file("${path.root}/${var.root_modules_path}/${module}/stacks/${yaml_file}")) + } + } + + ## Common Stack configurations + # Retrieve common Stack configurations for each root module + # Example: + # { + # "random-pet" = { + # "stack_settings" = { + # "description" = "This stack generates random pet names" + # "manage_state" = true + # } + # "tfvars" = { + # "enabled" = false + # } + # } + # } + + _common_configs = { + for module, files in local._root_module_yaml_decoded : module => lookup(files, var.common_config_file, {}) + } + + ## Stack Configurations + # Merge all Stack configurations from the root modules into a single map, and filter out the common config. + # Example: + # { + # "random-pet-example" = { + # "project_root" = "examples/complete/components/random-pet" + # "root_module" = "random-pet" + # "stack_settings" = { + # "manage_state" = true + # } + # "terraform_workspace" = "example" + # "tfvars" = { + # "enabled" = true + # } + # } + # } + + _root_module_stack_configs = merge([for module, files in local._root_module_yaml_decoded : { + for file, content in files : "${module}-${trimsuffix(file, ".yaml")}" => + merge( + { + "project_root" = replace(format("%s/%s", var.root_modules_path, module), "../", "") + "root_module" = module, + "terraform_workspace" = trimsuffix(file, ".yaml"), + }, + content + ) if file != var.common_config_file + } + ]...) + + # Get the configs for each stack, merged with the common configurations + # Example: + # { + # "random-pet-example" = { + # "project_root" = "examples/complete/components/random-pet" + # "root_module" = "random-pet" + # "stack_settings" = { + # "manage_state" = true + # } + # "terraform_workspace" = "example" + # "tfvars" = { + # "enabled" = false + # } + # } + # } + configs = { + for key, value in module.deep : key => value.merged + } + + # Get the Stacks configs, this is just to improve code readability + # Example: + # { + # "random-pet-example" = { + # "manage_state" = true + # } + # } + stack_configs = { + for key, value in local.configs : key => value.stack_settings + } + + # Get the list of all stack names + stacks = toset(keys(local.stack_configs)) + + ## Labels + # Сreates a map of administrative labels for each stack that has the administrative property set to true. + # Example: + # { + # "spacelift-automation-mp-main" = [ + # "administrative", + # ] + # "spacelift-policies-notify-tf-completed" = [ + # "administrative", + # ] + # } + + _administrative_labels = { + for stack, configs in local.stack_configs : stack => ["administrative"] if tobool(try(configs.administrative, false)) == true + } + + # Creates a map of `depends-on` labels for each stack based on the root module level dependency configuration. + # Example: + # { + # "random-pet-example" = [ + # "depends-on:spacelift-automation-default", + # ] + # } + + _dependency_labels = { + for stack in local.stacks : stack => [ + "depends-on:spacelift-automation-${terraform.workspace}" + ] + } + + # Creates a map of folder labels for each stack based on Git structure for a proper grouping stacks in Spacelift UI. + # https://docs.spacelift.io/concepts/stack/organizing-stacks#label-based-folders + # Example: + # { + # "random-pet-example" = [ + # "folder:random-pet/example", + # ] + # } + + _folder_labels = { + for stack in local.stacks : stack => [ + "folder:${local.configs[stack].root_module}/${local.configs[stack].terraform_workspace}" + ] + } + + # Merge all the labels into a single map for each stack. + # Example: + # { + # "random-pet-example" = tolist([ + # "folder:random-pet/example", + # "depends-on:spacelift-automation-default", + # ]) + # } + + labels = { + for stack in local.stacks : + stack => compact(flatten([ + lookup(local._administrative_labels, stack, []), + lookup(local._folder_labels, stack, []), + lookup(local._dependency_labels, stack, []), + try(local.stack_configs[stack].labels, []), + ])) + } + + # Merge all before_init steps into a single map for each stack. + before_init = { + for stack in local.stacks : stack => compact(concat( + var.before_init, + try(local.stack_configs[stack].before_init, []), + # This command is required for each stack. + # It copies the tfvars file from the stack's workspace to the root module's directory + # and renames it to `spacelift.auto.tfvars` to automatically load variable definitions for each run/task. + ["cp tfvars/${local.configs[stack].terraform_workspace}.tfvars spacelift.auto.tfvars"], + )) if try(local.configs[stack].tfvars.enabled, true) + } +} + +# Perform deep merge for common configurations and stack configurations +module "deep" { + source = "cloudposse/config/yaml//modules/deepmerge" + version = "1.0.2" + for_each = local._root_module_stack_configs + # Stack configuration will take precedence and overwrite the conflicting value from the common configuration (if any) + maps = [local._common_configs[each.value.root_module], each.value] +} + +resource "spacelift_stack" "default" { + for_each = local.stacks + + space_id = coalesce(try(local.stack_configs[each.key].space_id, null), var.space_id) + name = each.key + administrative = coalesce(try(local.stack_configs[each.key].administrative, null), var.administrative) + after_apply = compact(concat(try(local.stack_configs[each.key].after_apply, []), var.after_apply)) + after_destroy = compact(concat(try(local.stack_configs[each.key].after_destroy, []), var.after_destroy)) + after_init = compact(concat(try(local.stack_configs[each.key].after_init, []), var.after_init)) + after_perform = compact(concat(try(local.stack_configs[each.key].after_perform, []), var.after_perform)) + after_plan = compact(concat(try(local.stack_configs[each.key].after_plan, []), var.after_plan)) + autodeploy = coalesce(try(local.stack_configs[each.key].autodeploy, null), var.autodeploy) + autoretry = try(local.stack_configs[each.key].autoretry, var.autoretry) + before_apply = compact(coalesce(try(local.stack_configs[each.key].before_apply, []), var.before_apply)) + before_destroy = compact(coalesce(try(local.stack_configs[each.key].before_destroy, []), var.before_destroy)) + before_init = compact(coalesce(try(local.before_init[each.key], []), var.before_init)) + before_perform = compact(coalesce(try(local.stack_configs[each.key].before_perform, []), var.before_perform)) + before_plan = compact(coalesce(try(local.stack_configs[each.key].before_plan, []), var.before_plan)) + description = coalesce(try(local.stack_configs[each.key].description, null), var.description) + repository = try(local.stack_configs[each.key].repository, var.repository) + branch = try(local.stack_configs[each.key].branch, var.branch) + project_root = local.configs[each.key].project_root + manage_state = try(local.stack_configs[each.key].manage_state, var.manage_state) + labels = local.labels[each.key] + enable_local_preview = try(local.stack_configs[each.key].enable_local_preview, var.enable_local_preview) + terraform_smart_sanitization = try(local.stack_configs[each.key].terraform_smart_sanitization, var.terraform_smart_sanitization) + terraform_version = try(local.stack_configs[each.key].terraform_version, var.terraform_version) + terraform_workflow_tool = var.terraform_workflow_tool + terraform_workspace = local.configs[each.key].terraform_workspace + + protect_from_deletion = try(local.stack_configs[each.key].protect_from_deletion, var.protect_from_deletion) + + worker_pool_id = try(local.stack_configs[each.key].worker_pool_id, var.worker_pool_id) + + dynamic "github_enterprise" { + for_each = var.github_enterprise != null ? [var.github_enterprise] : [] + content { + namespace = github_enterprise.value["namespace"] + } + } +} + +# The Spacelift Destructor is a feature designed to automatically clean up the resources no longer managed by our IaC. +# Don't toggle the creation/destruction of this resource with var.destructor_enabled, +# as it will delete all resources in the stack when toggled from 'true' to 'false'. +# Use the 'deactivated' attribute to disable the stack destructor functionality instead. +# https://github.com/spacelift-io/terraform-provider-spacelift/blob/master/spacelift/resource_stack_destructor.go +resource "spacelift_stack_destructor" "default" { + for_each = local.stacks + + stack_id = spacelift_stack.default[each.key].id + deactivated = !try(local.stack_configs[each.key].destructor_enabled, var.destructor_enabled) + + # `depends_on` should be used to make sure that all necessary resources (environment variables, roles, integrations, etc.) + # are still in place when the destruction run is executed. + # See https://registry.terraform.io/providers/spacelift-io/spacelift/latest/docs/resources/stack_destructor + depends_on = [ + spacelift_drift_detection.default, + spacelift_aws_integration_attachment.default + ] +} + +resource "spacelift_aws_integration_attachment" "default" { + for_each = { + for stack, configs in local.stack_configs : stack => configs + if try(configs.aws_integration_enabled, var.aws_integration_enabled) + } + integration_id = try(local.stack_configs[each.key].aws_integration_id, var.aws_integration_id) + stack_id = spacelift_stack.default[each.key].id + read = var.aws_integration_attachment_read + write = var.aws_integration_attachment_write +} + +resource "spacelift_drift_detection" "default" { + for_each = { + for stack, configs in local.stack_configs : stack => configs + if try(configs.drift_detection_enabled, var.drift_detection_enabled) + } + + stack_id = spacelift_stack.default[each.key].id + ignore_state = try(local.stack_configs[each.key].drift_detection_ignore_state, var.drift_detection_ignore_state) + reconcile = try(local.stack_configs[each.key].drift_detection_reconcile, var.drift_detection_reconcile) + schedule = try(local.stack_configs[each.key].drift_detection_schedule, var.drift_detection_schedule) + timezone = try(local.stack_configs[each.key].drift_detection_timezone, var.drift_detection_timezone) + + lifecycle { + precondition { + condition = can(regex("^([0-9,\\-\\*]+\\s+){4}[0-9,\\-\\*]+$", try(local.stack_configs[each.key].drift_detection_schedule, var.drift_detection_schedule))) + error_message = "Invalid cron schedule format for drift detection" + } + } } diff --git a/outputs.tf b/outputs.tf deleted file mode 100644 index c44df14..0000000 --- a/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "random_pet_name" { - description = "The generated random pet name" - value = random_pet.template.id -} diff --git a/variables.tf b/variables.tf index 6348a57..801c1db 100644 --- a/variables.tf +++ b/variables.tf @@ -1,5 +1,259 @@ -variable "length" { - description = "The length of the random name" - type = number - default = 2 +# GitHub +variable "github_enterprise" { + type = object({ + namespace = string + id = optional(string) + }) + description = "The GitHub VCS settings" +} + +variable "repository" { + type = string + description = "The name of your infrastructure repo" +} + +variable "branch" { + type = string + description = "Specify which branch to use within the infrastructure repository." + default = "main" +} + +variable "root_modules_path" { + type = string + description = "The path, relative to the root of the repository, where the root module can be found." + default = "root-modules" +} + +variable "enabled_root_modules" { + type = list(string) + description = <<-EOT + List of root modules where to look for stack config files. + Ignored when all_root_modules_enabled is true. + Example: ["spacelift-automation", "k8s-cluster"] + EOT + default = [] +} + +variable "all_root_modules_enabled" { + type = bool + description = "When set to true, all subdirectories in root_modules_path will be treated as root modules." + default = false +} + +# Spacelift Backend +variable "terraform_workflow_tool" { + type = string + description = <<-EOT + Defines the tool that will be used to execute the workflow. + This can be one of OPEN_TOFU, TERRAFORM_FOSS or CUSTOM. + EOT + default = "OPEN_TOFU" + + validation { + condition = contains(["OPEN_TOFU", "TERRAFORM_FOSS", "CUSTOM"], var.terraform_workflow_tool) + error_message = "Valid values for terraform_workflow_tool are (OPEN_TOFU, TERRAFORM_FOSS, CUSTOM)." + } +} + +# Stack Cloud Integrations +variable "aws_integration_enabled" { + type = bool + description = "Indicates whether the AWS integration is enabled." + default = false +} + +variable "aws_integration_id" { + type = string + description = "ID of the AWS integration to attach." + default = null +} + +variable "aws_integration_attachment_read" { + type = bool + description = "Indicates whether this attachment is used for read operations." + default = true +} + +variable "aws_integration_attachment_write" { + type = bool + description = "Indicates whether this attachment is used for write operations." + default = true +} + +# Configuration for the Spacelift Stack +variable "common_config_file" { + type = string + description = "Name of the common configuration file for the stack across a root module." + default = "common.yaml" +} +# Default Stack Configuration +variable "administrative" { + type = bool + description = "Flag to mark the stack as administrative" + default = false +} + +variable "after_apply" { + type = list(string) + description = "List of after-apply scripts" + default = [] +} + +variable "after_destroy" { + type = list(string) + description = "List of after-destroy scripts" + default = [] +} + +variable "after_init" { + type = list(string) + description = "List of after-init scripts" + default = [] +} + +variable "after_perform" { + type = list(string) + description = "List of after-perform scripts" + default = [] +} + +variable "after_plan" { + type = list(string) + description = "List of after-plan scripts" + default = [] +} + +variable "autodeploy" { + type = bool + description = "Flag to enable/disable automatic deployment of the stack" + default = true +} + +variable "autoretry" { + type = bool + description = "Flag to enable/disable automatic retry of the stack" + default = false +} + +variable "before_apply" { + type = list(string) + description = "List of before-apply scripts" + default = [] +} + +variable "before_destroy" { + type = list(string) + description = "List of before-destroy scripts" + default = [] +} + +variable "before_init" { + type = list(string) + description = "List of before-init scripts" + default = [] +} + +variable "before_perform" { + type = list(string) + description = "List of before-perform scripts" + default = [] +} + +variable "before_plan" { + type = list(string) + description = "List of before-plan scripts" + default = [] +} + +variable "description" { + type = string + description = "Description of the stack" + default = "Managed by spacelift-automation Terraform root module." +} + +variable "destructor_enabled" { + type = bool + description = "Flag to enable/disable the destructor for the Stack." + default = false +} + +variable "drift_detection_enabled" { + type = bool + description = "Flag to enable/disable Drift Detection configuration for a Stack." + default = false +} + +variable "drift_detection_ignore_state" { + type = bool + description = <<-EOT + Controls whether drift detection should be performed on a stack + in any final state instead of just 'Finished'. + EOT + default = false +} + +variable "drift_detection_reconcile" { + type = bool + description = "Flag to enable/disable automatic reconciliation of drifts." + default = false +} + +variable "drift_detection_schedule" { + type = list(string) + description = "The schedule for drift detection." + default = ["0 4 * * *"] +} + +variable "drift_detection_timezone" { + type = string + description = "The timezone for drift detection." + default = "UTC" +} + +variable "enable_local_preview" { + type = bool + description = "Indicates whether local preview runs can be triggered on this Stack." + default = false + +} +variable "manage_state" { + type = bool + description = "Determines if Spacelift should manage state for this stack." + default = false +} + +variable "protect_from_deletion" { + type = bool + description = "Protect this stack from accidental deletion. If set, attempts to delete this stack will fail." + default = false +} + +variable "space_id" { + type = string + description = "Place the stack in the specified space_id." + default = "root" +} + +variable "terraform_smart_sanitization" { + type = bool + description = <<-EOT + Indicates whether runs on this will use terraform's sensitive value system to sanitize + the outputs of Terraform state and plans in spacelift instead of sanitizing all fields. + EOT + default = false +} + +variable "terraform_version" { + type = string + description = "Terraform version to use." + default = "1.7.2" +} + +variable "worker_pool_id" { + type = string + description = <<-EOT + ID of the worker pool to use. + NOTE: worker_pool_id is required when using a self-hosted instance of Spacelift. + EOT + default = null } diff --git a/versions.tf b/versions.tf index 0cf661c..5d935ac 100644 --- a/versions.tf +++ b/versions.tf @@ -1,10 +1,10 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.6" required_providers { - random = { - source = "hashicorp/random" - version = ">= 3.0" + spacelift = { + source = "spacelift-io/spacelift" + version = ">= 1.14" } } }