Secrets manager plus usage in RDS create

.gitignore

@@ -5,3 +5,9 @@
 .DS_Store
 docs
 inst/doc
+
+# tests fixtures
+tests/fixtures/aws_db_rds_create.yml
+tests/fixtures/aws_db_redshift_create.yml
+tests/fixtures/aws_secret*.yml
+tests/fixtures/aws_db_rds_create.yml

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: sixtyfour
 Title: Humane Interface to AWS
-Version: 0.0.0.97
+Version: 0.0.0.98
 Authors@R: c(
     person("Sean", "Kross", role = "aut"),
     person("Scott", "Chamberlain", role = c("aut", "cre"), email = "sachamber@fredhutch.org")
@@ -28,7 +28,10 @@ Imports:
     s3fs (>= 0.1.3),
     cli,
     glue,
-    memoise
+    memoise,
+    uuid,
+    jsonlite,
+    curl
 Suggests: 
     knitr,
     rmarkdown,

Makefile

@@ -40,3 +40,10 @@ style_file:
 
 style_package:
 	${RSCRIPT} -e "styler::style_pkg()"
+
+scan_secrets:
+	@echo "scanning for leaks in commits\n"
+	gitleaks detect --source . -v
+	@echo "\n\n\n"
+	@echo "scanning for leaks in uncommitted files\n"
+	gitleaks protect --source . -v

NAMESPACE

@@ -14,9 +14,11 @@ export(aws_db_instance_status)
 export(aws_db_rds_client)
 export(aws_db_rds_con)
 export(aws_db_rds_create)
+export(aws_db_rds_list)
 export(aws_db_redshift_client)
 export(aws_db_redshift_con)
 export(aws_db_redshift_create)
+export(aws_ec2_client)
 export(aws_file_attr)
 export(aws_file_copy)
 export(aws_file_delete)
@@ -33,21 +35,41 @@ export(aws_iam_client)
 export(aws_policies)
 export(aws_policy)
 export(aws_policy_attach)
+export(aws_policy_create)
 export(aws_policy_detach)
+export(aws_policy_document_create)
 export(aws_policy_exists)
 export(aws_role)
 export(aws_role_create)
 export(aws_role_delete)
 export(aws_roles)
+export(aws_secrets_all)
+export(aws_secrets_create)
+export(aws_secrets_delete)
+export(aws_secrets_get)
+export(aws_secrets_list)
+export(aws_secrets_pwd)
+export(aws_secrets_rotate)
+export(aws_secrets_update)
 export(aws_user)
 export(aws_user_access_key)
 export(aws_user_add_to_group)
+export(aws_user_add_to_rds)
 export(aws_user_create)
 export(aws_user_current)
 export(aws_user_delete)
 export(aws_user_exists)
 export(aws_users)
+export(aws_vpc)
+export(aws_vpc_security_group)
+export(aws_vpc_security_group_create)
+export(aws_vpc_security_group_ingress)
+export(aws_vpc_security_group_modify_rules)
+export(aws_vpc_security_groups)
+export(aws_vpcs)
 export(billing)
+export(ip_permissions_generator)
+export(random_user)
 export(s3_path)
 export(set_s3_interface)
 importFrom(cli,cli_inform)
@@ -56,22 +78,31 @@ importFrom(cli,cli_progress_update)
 importFrom(cli,pb_spin)
 importFrom(dplyr,bind_rows)
 importFrom(dplyr,filter)
+importFrom(dplyr,last_col)
 importFrom(dplyr,mutate)
 importFrom(dplyr,pull)
+importFrom(dplyr,relocate)
+importFrom(dplyr,select)
 importFrom(fs,file_exists)
 importFrom(fs,fs_bytes)
 importFrom(glue,glue)
+importFrom(jsonlite,fromJSON)
+importFrom(jsonlite,toJSON)
 importFrom(lubridate,as_datetime)
+importFrom(magrittr,"%<>%")
 importFrom(magrittr,"%>%")
 importFrom(paws,costexplorer)
 importFrom(paws,iam)
 importFrom(paws,rds)
 importFrom(paws,redshift)
 importFrom(paws,s3)
 importFrom(purrr,flatten)
+importFrom(purrr,keep)
 importFrom(purrr,list_rbind)
 importFrom(purrr,map)
 importFrom(purrr,map_chr)
+importFrom(purrr,map_lgl)
+importFrom(purrr,pluck)
 importFrom(rlang,":=")
 importFrom(rlang,has_name)
 importFrom(s3fs,s3_dir_info)

R/database-misc.R

@@ -1,12 +1,14 @@
 #' internal helper function
-#' @param id an RDS instance ID or Redshift cluster ID
-#' @param fun a function that takes an ID for an AWS RDS instance
+#' @param id (function) an RDS instance ID or Redshift cluster ID
+#' @param fun (function) a function that takes an ID for an AWS RDS instance
 #' or Redshift cluster, and returns a single boolean
+#' @param see_fun (character) the function to point users to in the
+#' message for database connection
 #' @noRd
 #' @keywords internal
-info <- function(id, fun) {
+info <- function(id, fun, see_fun = "") {
   cli::cli_alert_success("Instance is up!")
-  cli::cli_alert_info("See `aws_db_rds_con` for connection info")
+  cli::cli_alert_info("See `{see_fun}` for connection info")
   cli::cli_alert_info("Instance details:")
   con_info <- fun(id)
   for (i in seq_along(con_info)) {
@@ -32,3 +34,7 @@ which_driver <- function(engine) {
     stop(glue::glue("{engine} not currently supported"))
   )
 }
+
+random_str <- function(prefix = "-") {
+  paste0(prefix, sub("-.+", "", uuid::UUIDgenerate()))
+}

R/database-rds.R

@@ -1,13 +1,13 @@
 #' Get a database connection to Amazon RDS
 #'
-#' Supports: MariaDB, MySQL, and PostgreSQL
+#' Supports: MariaDB, MySQL, and Postgres
 #'
 #' @export
 #' @inheritParams aws_db_redshift_con
 #' @param engine (character) The engine to use. optional if `user`, `pwd`, and
 #' `id` are supplied - otherwise required
 #' @details RDS supports many databases, but we only provide support for
-#' MariaDB, MySQL, and PostgreSQL
+#' MariaDB, MySQL, and Postgres
 #'
 #' If the `engine` you've chosen for your RDS instance is not supported
 #' with this function, you can likely connect to it on your own
@@ -26,12 +26,11 @@
 #' library(dplyr)
 #' tbl(con_rds, "mtcars")
 #' }
-aws_db_rds_con <- function(user, pwd, id = NULL, host = NULL, port = NULL,
-                           dbname = NULL, engine = NULL, ...) {
+aws_db_rds_con <- function(
+    user = NULL, pwd = NULL, id = NULL, host = NULL,
+    port = NULL, dbname = NULL, engine = NULL, ...) {
   check_for_pkg("DBI")
-
-  stopifnot("user is required" = !missing(user))
-  stopifnot("pwd is required" = !missing(pwd))
+  is_class(engine, "character")
 
   if (!is.null(id)) {
     con_info <- instance_con_info(id)
@@ -46,13 +45,15 @@
     )
   }
 
+  creds <- ui_fetch_secret(user, pwd, engine)
+
   DBI::dbConnect(
     which_driver(engine),
     host = host,
     port = port,
     dbname = dbname,
-    user = user,
-    password = pwd,
+    user = creds$user,
+    password = creds$password,
     ...
   )
 }
@@ -64,18 +65,21 @@
 #' @param id (character) required. instance identifier. The identifier for
 #' this DB instance. This parameter is stored as a lowercase string.
 #' Constraints: must contain from 1 to 63 letters, numbers, or hyphens; first
-#' character must be a letter; cn't end with a hyphen or contain two
+#' character must be a letter; can't end with a hyphen or contain two
 #' consecutive hyphens. required.
 #' @param class (character) required. The compute and memory capacity of the
 #' DB instance, for example `db.m5.large`.
 #' @param user (character) User name associated with the admin user account for
-#' the cluster that is being created.
+#' the cluster that is being created. If `NULL`, we generate a random user
+#' name, see [random_user()]
 #' @param pwd (character) Password associated with the admin user account for
-#' the cluster that is being created.
+#' the cluster that is being created. If `NULL`, we generate a random password
+#' with [aws_secrets_pwd()] (which uses the AWS Secrets Manager service)
 #' @param dbname (character) The name of the first database to be created when
 #' the cluster is created. default: "dev". additional databases can be created
 #' within the cluster
 #' @param engine (character) The engine to use. default: "mariadb". required.
+#' one of: mariadb, mysql, or postgres
 #' @param storage (character) The amount of storage in gibibytes (GiB) to
 #' allocate for the DB instance. default: 20
 #' @param storage_encrypted (logical) Whether the DB instance is encrypted.
@@ -88,6 +92,10 @@
 #' until the cluster is available. If `wait=FALSE` use
 #' `aws_db_instance_status()` to check on the cluster status.
 #' @param verbose (logical) verbose informational output? default: `TRUE`
+#' @param aws_secrets (logical) should we manage your database credentials
+#' in AWS Secrets Manager? default: `TRUE`
+#' @param iam_database_auth (logical) Use IAM database authentication?
+#' default: `FALSE`
 #' @param ... named parameters passed on to
 #' [create_db_instance](https://www.paws-r-sdk.com/docs/rds_create_db_instance/)
 #' @details See above link to `create_cluster` docs for details on requirements
@@ -102,29 +110,58 @@
 #' instead set `wait = FALSE` and then check on the status of the instance
 #' yourself in the AWS dashboard.
 #' @family database
-#' @return a list with methods for interfacing with RDS;
-#' see <https://www.paws-r-sdk.com/docs/rds/>. also prints useful
-#' connection information after instance is available.
+#' @return returns `NULL`, this function called for the side effect of
+#' creating an RDS instance
 aws_db_rds_create <-
-  function(id, class, user, pwd, dbname = "dev",
+  function(id, class, user = NULL, pwd = NULL, dbname = "dev",
            engine = "mariadb", storage = 20,
            storage_encrypted = TRUE, security_group_ids = NULL,
-           wait = TRUE, verbose = TRUE, ...) {
+           wait = TRUE, verbose = TRUE, aws_secrets = TRUE,
+           iam_database_auth = FALSE,...) {
     aws_db_rds_client()
+    if (is.null(user)) {
+      user <- random_user()
+      if (verbose) {
+        cli::cli_alert_info("`user` is NULL; created user: {.strong {user}}")
+      }
+    }
+    if (is.null(pwd)) {
+      pwd <- aws_secrets_pwd()
+      if (verbose) {
+        cli::cli_alert_info("`pwd` is NULL; created password: *******")
+      }
+    }
+    security_group_ids <- security_group_handler(security_group_ids, engine)
     env64$rds$create_db_instance(
       DBName = dbname, DBInstanceIdentifier = id,
       Engine = engine, DBInstanceClass = class,
       AllocatedStorage = storage,
       MasterUsername = user, MasterUserPassword = pwd,
       VpcSecurityGroupIds = security_group_ids,
       StorageEncrypted = storage_encrypted,
+      EnableIAMDatabaseAuthentication = iam_database_auth,
       ...
     )
     if (wait) {
       wait_for_instance(id)
     }
-    if (verbose) info(id, instance_con_info)
-    return(env64$rds)
+    if (aws_secrets) {
+      if (verbose) cli::cli_alert_info("Uploading user/pwd to secrets manager")
+      x <- instance_con_info(id)
+      aws_secrets_create(
+        name = paste0(id, random_str()),
+        secret = construct_db_secret(
+          engine = x$engine,
+          host = x$host,
+          username = user,
+          password = pwd,
+          dbname = x$dbname,
+          port = x$port
+        )
+      )
+    }
+    if (verbose) info(id, instance_con_info, "aws_db_rds_con")
+    invisible()
   }
 
 #' Get the `paws` RDS client
@@ -147,13 +184,49 @@
   return(instances)
 }
 
+split_grep <- function(column, split, pattern) {
+  grep(glue("^{pattern}$"), strsplit(column, split)[[1]], value = TRUE)
+}
+
+#' Get information for all RDS instances
+#' @importFrom dplyr select
+#' @export
+#' @family database
+#' @return a tibble of instance details;
+#' see <https://www.paws-r-sdk.com/docs/describe_db_instances/>
+#' @autoglobal
+#' @examplesIf interactive()
+#' aws_db_rds_list()
+aws_db_rds_list <- function() {
+  lst <- instance_details()
+  dbs <- lst$DBInstances
+  map(dbs, \(x) as_tibble(x[c(
+    "DBInstanceIdentifier",
+    "DBInstanceClass",
+    "Engine",
+    "DBInstanceStatus",
+    "DBName",
+    "DbiResourceId",
+    "DBInstanceArn"
+  )])) %>%
+    list_rbind() %>%
+    mutate(
+      AccountId = split_grep(DBInstanceArn, ":", "^[0-9]+$"),
+      Region = split_grep(DBInstanceArn, ":", "^[a-z]+-[a-z]+-[0-9]$")
+    ) %>%
+    select(-DBInstanceArn)
+}
+
 #' Get connection information for all instances
+#' @importFrom purrr keep
 #' @inheritParams aws_db_redshift_create
 #' @return a list of cluster details
 #' @keywords internal
 instance_con_info <- function(id) {
   deets <- instance_details()$DBInstances
-  z <- Filter(function(x) x$DBInstanceIdentifier == id, deets)[[1]]
+  z <- keep(deets, \(x) x$DBInstanceIdentifier == id)
+  if (!length(z)) rlang::abort(glue("Instance identifier {id} not found"))
+  z <- z[[1]]
   list(
     host = z$Endpoint$Address,
     port = z$Endpoint$Port,

R/database-redshift.R

@@ -95,9 +95,8 @@ aws_db_redshift_con <- function(user, pwd, id = NULL, host = NULL, port = NULL,
 #' for each parameter
 #' @inheritSection aws_db_rds_create Waiting
 #' @family database
-#' @return a list with methods for interfacing with Redshift;
-#' see <https://www.paws-r-sdk.com/docs/redshift/>. also prints useful
-#' connection information after cluster is available.
+#' @return returns `NULL`, this function called for the side effect of
+#' creating an Redshift instance
 aws_db_redshift_create <-
   function(id, user, pwd, dbname = "dev", cluster_type = "multi-node",
            node_type = "dc2.large", number_nodes = 2,
@@ -114,8 +113,8 @@ aws_db_redshift_create <-
     if (wait) {
       wait_for_cluster(id)
     }
-    if (verbose) info(id, cluster_con_info)
-    return(env64$redshift)
+    if (verbose) info(id, cluster_con_info, "aws_db_redshift_con")
+    invisible()
   }
 
 #' Get the `paws` Redshift client