Merge pull request #48 from getwilds/minio

Minio for testing file and bucket fxns

.Rbuildignore

@@ -9,3 +9,4 @@
 ^README\.Rmd$
 ^data-raw$
 vignettes/figure
+start/

.github/workflows/R-CMD-check.yaml

@@ -1,9 +1,10 @@
 # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
 on:
   push:
-    branches: [main]
+    branches: [main, minio]
   pull_request:
     branches: [main]
+  workflow_dispatch:
 
 name: R-CMD-check
 
@@ -26,6 +27,9 @@ jobs:
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
       R_KEEP_PKG_SOURCE: yes
+      MINIO_USER: ${{ secrets.MINIO_USER }}
+      MINIO_PWD: ${{ secrets.MINIO_PWD }}
+      MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }}
 
     steps:
       - uses: actions/checkout@v3
@@ -43,6 +47,14 @@ jobs:
           extra-packages: any::rcmdcheck
           needs: check
 
+      - name: Start Minio
+        if: runner.os == 'Linux'
+        run: |
+          wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb
+          sudo dpkg -i minio.deb
+          mkdir ~/minio
+          minio server ~/minio --console-address :9001 &
+
       - uses: r-lib/actions/check-r-package@v2
         with:
           upload-snapshots: true

.github/workflows/test-coverage.yaml

@@ -0,0 +1,61 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+on:
+  push:
+    branches: [main]
+
+name: test-coverage
+
+jobs:
+  test-coverage:
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      MINIO_USER: ${{ secrets.MINIO_USER }}
+      MINIO_PWD: ${{ secrets.MINIO_PWD }}
+      MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::covr
+          needs: coverage
+
+      - name: Start Minio
+        run: |
+          wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb
+          sudo dpkg -i minio.deb
+          mkdir ~/minio
+          minio server ~/minio --console-address :9001 &
+
+      - name: Test coverage
+        run: |
+          covr::codecov(
+            quiet = FALSE,
+            clean = FALSE,
+            install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package")
+          )
+        shell: Rscript {0}
+        env:
+          AWS_REGION: a
+          AWS_ACCESS_KEY_ID: b
+          AWS_SECRET_ACCESS_KEY: c
+
+      - name: Show testthat output
+        if: always()
+        run: |
+          ## --------------------------------------------------------------------
+          find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true
+        shell: bash
+
+      - name: Upload test results
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-test-failures
+          path: ${{ runner.temp }}/package

.gitignore

@@ -10,3 +10,24 @@ inst/doc
 tests/fixtures/aws_db_rds_create.yml
 tests/fixtures/aws_db_redshift_create.yml
 tests/fixtures/aws_secret*.yml
+tests/fixtures/aws_bucket_add_user.yml
+tests/fixtures/aws_bucket_add_user_setup_bucket.yml
+tests/fixtures/aws_bucket_add_user_setup_user.yml
+tests/fixtures/aws_bucket_get_permissions.yml
+tests/fixtures/aws_bucket_get_permissions_setup_bucket.yml
+tests/fixtures/aws_bucket_get_permissions_setup_users.yml
+tests/fixtures/aws_bucket_permissions.yml
+tests/fixtures/aws_bucket_permissions_setup_bucket.yml
+tests/fixtures/aws_bucket_permissions_setup_users.yml
+tests/fixtures/aws_bucket_remove_user.yml
+tests/fixtures/aws_bucket_remove_user_setup_add_user.yml
+tests/fixtures/aws_bucket_remove_user_setup_bucket.yml
+tests/fixtures/aws_bucket_remove_user_setup_user.yml
+tests/fixtures/aws_policy_attach_setup_users.yml
+tests/fixtures/aws_policy_detach_setup_attach.yml
+tests/fixtures/aws_policy_detach_setup_user.yml
+tests/fixtures/aws_policy_list_entities_empty.yml
+tests/fixtures/aws_policy_list_entities_non_empty.yml
+
+# minio
+start/

Makefile

@@ -65,3 +65,9 @@ scan_secrets:
 	@echo "\n\n\n"
 	@echo "scanning for leaks in uncommitted files\n"
 	gitleaks protect --source . -v
+
+minio_start:
+	MINIO_USER=${MINIO_USER} ;\
+	MINIO_PWD=${MINIO_PWD} ;\
+	MINIO_ENDPOINT=${MINIO_ENDPOINT} ;\
+	minio server /tmp/minio --console-address :9090

NAMESPACE

@@ -73,6 +73,7 @@ export(ip_permissions_generator)
 export(random_user)
 export(s3_path)
 export(set_s3_interface)
+importFrom(cli,cli_abort)
 importFrom(cli,cli_inform)
 importFrom(cli,cli_progress_bar)
 importFrom(cli,cli_progress_update)

R/bucket.R

@@ -1,17 +1,20 @@
-#' Create an S3 bucket
+#' Check if an S3 bucket exists
 #'
 #' @export
-#' @param bucket (character) bucket name. required
+#' @param bucket (character) bucket name; must be length 1. required
 #' @note internally uses
 #' [head_bucket](https://www.paws-r-sdk.com/docs/s3_head_bucket/)
 #' @family buckets
+#' @return a single boolean (logical)
 #' @examples \dontrun{
 #' # exists
 #' aws_bucket_exists(bucket = "s64-test-2")
 #' # does not exist
 #' aws_bucket_exists(bucket = "no-bucket")
 #' }
 aws_bucket_exists <- function(bucket) {
+  stop_if_not(rlang::is_character(bucket), "bucket must be character")
+  stop_if_not(length(bucket) == 1, "length(bucket) != 1")
   res <- tryCatch(
     {
       env64$s3$head_bucket(Bucket = bucket)
@@ -26,14 +29,16 @@ aws_bucket_exists <- function(bucket) {
 #' @export
 #' @param bucket (character) bucket name. required
 #' @param ... named parameters passed on to
-#' [list_objects](https://www.paws-r-sdk.com/docs/s3_create_bucket/)
+#' [create_bucket](https://www.paws-r-sdk.com/docs/s3_create_bucket/)
 #' @note Requires the env var `AWS_REGION`
 #' @return the bucket path (character)
 #' @family buckets
 #' @examples \dontrun{
 #' aws_bucket_create(bucket = "s64-test-2")
 #' }
 aws_bucket_create <- function(bucket, ...) {
+  stop_if_not(rlang::is_character(bucket), "bucket must be character")
+  stop_if_not(length(bucket) == 1, "length(bucket) != 1")
   env64$s3$create_bucket(
     Bucket = bucket,
     CreateBucketConfiguration =
@@ -80,6 +85,8 @@ bucket_create_if_not <- function(bucket, force = FALSE) {
 #'   aws_buckets()
 #' }
 aws_bucket_delete <- function(bucket, force = FALSE, ...) {
+  stop_if_not(rlang::is_character(bucket), "bucket must be character")
+  stop_if_not(length(bucket) == 1, "length(bucket) != 1")
   # TODO: add a package level option to override the prompt for adv. users
   if (!force) {
     if (yesno("Are you sure you want to delete {.strong {bucket}}?")) {
@@ -118,10 +125,10 @@ aws_bucket_download <- function(bucket, dest_path, ...) {
 #' @export
 #' @importFrom fs fs_bytes
 #' @param path (character) local path to a directory. required
-#' @param bucket (character) bucket name. required
 #' @param max_batch (fs_bytes) maximum batch size being uploaded with each
 #' multipart
 #' @param ... named parameters passed on to [s3fs::s3_dir_upload()]
+#' @inheritParams aws_bucket_delete
 #' @note Requires the env var `AWS_REGION`. This function prompts you to make
 #' sure that you want to delete the bucket.
 #' @family buckets
@@ -144,18 +151,24 @@ aws_bucket_download <- function(bucket, dest_path, ...) {
 #' aws_bucket_delete(bucket_name, force = TRUE)
 #' aws_bucket_exists(bucket_name)
 aws_bucket_upload <- function(
-    path, bucket, max_batch = fs::fs_bytes("100MB"),
+    path, bucket, max_batch = fs::fs_bytes("100MB"), force = FALSE,
     ...) {
+  stop_if(rlang::is_missing(path), "{.strong path} is required")
+  stop_if(rlang::is_missing(bucket), "{.strong bucket} is required")
   if (!aws_bucket_exists(bucket)) {
-    if (yesno("{.strong {bucket}} does not exist. Create it?")) {
-      cli::cli_inform("Exiting without uploading {.strong {basename(path)}}")
-      return(invisible())
+    if (!force) {
+      if (yesno("{.strong {bucket}} does not exist. Create it?")) {
+        cli::cli_inform("Exiting without uploading {.strong {basename(path)}}")
+        return(invisible())
+      }
     }
     aws_bucket_create(bucket)
   }
   s3fs::s3_dir_upload(
-    path = path, new_path = bucket,
-    max_batch = max_batch
+    path = path,
+    new_path = bucket,
+    max_batch = max_batch,
+    ...
   )
 }
 
@@ -199,6 +212,7 @@ aws_bucket_list_objects <- function(bucket, ...) {
 #' @export
 #' @importFrom s3fs s3_dir_info
 #' @inherit aws_bucket_list_objects
+#' @details internally uses [s3fs::s3_dir_info()]
 #' @note we set `refresh=TRUE` internally to make sure we return up to date
 #' information about your buckets rather than what's cached locally
 #' @family buckets

R/files.R

@@ -64,11 +64,11 @@ aws_file_upload <- function(path, remote_path, force = FALSE, ...) {
 #' Download a file
 #'
 #' @export
+#' @importFrom cli cli_abort
 #' @param remote_path (character) one or more remote S3 paths. required
 #' @param path (character) one or more file paths to write to. required
 #' @param ... named parameters passed on to [s3fs::s3_file_download()]
 #' @return (character) a vector of local file paths
-#' @note USES A FORK OF s3fs FOR A MINOR FIX THAT MAKES LENGTH>1 INPUTS WORK
 #' @family files
 #' @examples \dontrun{
 #' tfile <- tempfile()
@@ -94,7 +94,18 @@ aws_file_upload <- function(path, remote_path, force = FALSE, ...) {
 #' }
 aws_file_download <- function(remote_path, path, ...) {
   equal_lengths(remote_path, path)
-  s3fs::s3_file_download(remote_path, path, ...)
+  res <- tryCatch(
+    s3fs::s3_file_download(remote_path, path),
+    error = function(e) e
+  )
+  if (rlang::is_error(res)) {
+    if (grepl("SerializationError", res$message)) {
+      cli::cli_abort(c("Remote file not found", "S3 error: {res$message}"))
+    } else {
+      cli::cli_abort(res$message)
+    }
+  }
+  res
 }
 
 #' Delete a file
@@ -118,7 +129,16 @@ aws_file_download <- function(remote_path, path, ...) {
 #' aws_file_delete(s3_path("s64-test-2", "TESTING123"))
 #' }
 aws_file_delete <- function(remote_path, ...) {
-  s3fs::s3_file_delete(remote_path, ...)
+  # FIXME: this s3fs fxn not working for some reason, not sure why yet
+  # using paws for now
+  # s3fs::s3_file_delete(remote_path, ...)
+  path_parsed <- path_s3_parse(remote_path)
+  key <- if (nchar(path_parsed[[1]]$path)) {
+    file.path(path_parsed[[1]]$path, path_parsed[[1]]$file)
+  } else {
+    path_parsed[[1]]$file
+  }
+  env64$s3$delete_object(path_parsed[[1]]$bucket, key)
 }
 
 #' File attributes
@@ -217,6 +237,8 @@ aws_file_rename <- function(remote_path, new_remote_path, ...) {
 #' aws_file_copy(paths, "s64-test-4")
 #' }
 aws_file_copy <- function(remote_path, bucket, force = FALSE, ...) {
+  stop_if(rlang::is_missing(remote_path), "{.strong remote_path} is required")
+  stop_if(rlang::is_missing(bucket), "{.strong bucket} is required")
   bucket_create_if_not(bucket, force)
   parsed <- path_s3_parse(remote_path)
   parsed <- purrr::map(parsed, function(x) {

R/utils.R

@@ -222,8 +222,8 @@ is_class <- function(x, class) {
 }
 
 stop_if_not <- function(cond, msg) {
-  if (!cond) rlang::abort(msg)
+  if (!cond) cli::cli_abort(msg)
 }
 stop_if <- function(cond, msg) {
-  if (cond) rlang::abort(msg)
+  if (cond) cli::cli_abort(msg)
 }

README.Rmd

@@ -3,6 +3,7 @@
 <!-- badges: start -->
 [![Project Status: Prototype – Useable, some support, open to feedback, unstable API.](https://getwilds.org/badges/badges/prototype.svg)](https://getwilds.org/badges/#prototype)
 [![R-CMD-check](https://github.com/getwilds/sixtyfour/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/getwilds/sixtyfour/actions/workflows/R-CMD-check.yaml)
+[![codecov](https://codecov.io/gh/getwilds/sixtyfour/graph/badge.svg?token=BMER9MWIDN)](https://codecov.io/gh/getwilds/sixtyfour)
 <!-- badges: end -->
 
 A science-focused, more humane R interface to AWS.
@@ -30,6 +31,7 @@ Open an issue on our [issue tracker](https://github.com/getwilds/sixtyfour/issue
 
 - See the targets in the Makefile for various package maintenance tasks.
 - For scanning for secrets see the make target `scan_secrets` in the Makefile.
+- We use [minio][] for testing functions in this package (start with `aws_bucket`, and `aws_file`) for interacting with S3. See the make target `minio_start` for starting minio locally. The `R-CMD-check` workflow in `.github/workflows/` includes spinning up minio for running unit tests - only on linux. If you run tests and minio is not available then tests that require it will be skipped.
 
 ## Code of Conduct
 
@@ -38,3 +40,4 @@ Please note that the sixtyfour project is released with a [Contributor Code of C
 
 [paws]: https://www.paws-r-sdk.com/
 [s3fs]: https://dyfanjones.github.io/s3fs/
+[minio]: https://min.io/

README.md

@@ -3,6 +3,7 @@
 <!-- badges: start -->
 [![Project Status: Prototype – Useable, some support, open to feedback, unstable API.](https://getwilds.org/badges/badges/prototype.svg)](https://getwilds.org/badges/#prototype)
 [![R-CMD-check](https://github.com/getwilds/sixtyfour/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/getwilds/sixtyfour/actions/workflows/R-CMD-check.yaml)
+[![codecov](https://codecov.io/gh/getwilds/sixtyfour/graph/badge.svg?token=BMER9MWIDN)](https://codecov.io/gh/getwilds/sixtyfour)
 <!-- badges: end -->
 
 A science-focused, more humane R interface to AWS.
@@ -31,6 +32,7 @@ Open an issue on our [issue tracker](https://github.com/getwilds/sixtyfour/issue
 
 - See the targets in the Makefile for various package maintenance tasks.
 - For scanning for secrets see the make target `scan_secrets` in the Makefile.
+- We use [minio][] for testing functions in this package (start with `aws_bucket`, and `aws_file`) for interacting with S3. See the make target `minio_start` for starting minio locally. The `R-CMD-check` workflow in `.github/workflows/` includes spinning up minio for running unit tests - only on linux. If you run tests and minio is not available then tests that require it will be skipped.
 
 ## Code of Conduct
 
@@ -39,3 +41,4 @@ Please note that the sixtyfour project is released with a [Contributor Code of C
 
 [paws]: https://www.paws-r-sdk.com/
 [s3fs]: https://dyfanjones.github.io/s3fs/
+[minio]: https://min.io/