From 7e06b9ac110516aca5227de6670abb56693482da Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Fri, 22 Mar 2024 22:44:06 -0700 Subject: [PATCH 01/20] trying minio --- .Rbuildignore | 1 + .github/workflows/R-CMD-check.yaml | 14 +++++++++++ .gitignore | 3 +++ Makefile | 6 +++++ tests/testthat/helper-minio.R | 16 +++++++++++++ tests/testthat/test-db-rds.R | 2 +- tests/testthat/test-files.R | 37 ++++++++++++++++++++++++++++++ 7 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 tests/testthat/helper-minio.R create mode 100644 tests/testthat/test-files.R diff --git a/.Rbuildignore b/.Rbuildignore index d5c61b9..06fb1d9 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -9,3 +9,4 @@ ^README\.Rmd$ ^data-raw$ vignettes/figure +start/ diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index bbea2e2..bdfd636 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -43,6 +43,17 @@ jobs: extra-packages: any::rcmdcheck needs: check + - name: Start Minio + run: | + wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb + sudo dpkg -i minio.deb + mkdir ~/minio + minio server ~/minio --console-address :9001 + env: + MINIO_USER: ${{ secrets.MINIO_USER }} + MINIO_PWD: ${{ secrets.MINIO_PWD }} + MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} + - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true @@ -50,3 +61,6 @@ jobs: AWS_REGION: a AWS_ACCESS_KEY_ID: b AWS_SECRET_ACCESS_KEY: c + MINIO_USER: ${{ secrets.MINIO_USER }} + MINIO_PWD: ${{ secrets.MINIO_PWD }} + MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} diff --git a/.gitignore b/.gitignore index 14e538f..9176a7a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ inst/doc tests/fixtures/aws_db_rds_create.yml tests/fixtures/aws_db_redshift_create.yml tests/fixtures/aws_secret*.yml + +# minio +start/ diff --git a/Makefile b/Makefile index 5df03cd..30348db 100644 --- a/Makefile +++ b/Makefile @@ -65,3 +65,9 @@ scan_secrets: @echo "\n\n\n" @echo "scanning for leaks in uncommitted files\n" gitleaks protect --source . -v + +minio_start: + MINIO_USER=${MINIO_USER} ;\ + MINIO_PWD=${MINIO_PWD} ;\ + MINIO_ENDPOINT=${MINIO_ENDPOINT} ;\ + minio server start --console-address :9090 diff --git a/tests/testthat/helper-minio.R b/tests/testthat/helper-minio.R new file mode 100644 index 0000000..e0426f8 --- /dev/null +++ b/tests/testthat/helper-minio.R @@ -0,0 +1,16 @@ +bucket_delete <- function(bucket, force = FALSE) { + if (!aws_bucket_exists(bucket)) return() + list_obs <- purrr::safely(aws_bucket_list_objects) + objects <- list_obs(bucket) + if (NROW(objects$result) > 0) { + purrr::map(objects$result$uri, \(x) aws_file_delete(x)) + } + aws_bucket_delete(bucket, force = force) +} + +buckets_empty <- function() { + buckets <- aws_buckets() + if (NROW(buckets) > 0) { + invisible(purrr::map(buckets$bucket_name, bucket_delete, force = TRUE)) + } +} diff --git a/tests/testthat/test-db-rds.R b/tests/testthat/test-db-rds.R index 7c266c2..bf9830a 100644 --- a/tests/testthat/test-db-rds.R +++ b/tests/testthat/test-db-rds.R @@ -2,7 +2,7 @@ test_that("aws_db_rds_create", { skip_on_ci() vcr::use_cassette("aws_db_rds_create", { z <- aws_db_rds_create( - id = "bananas", class = "db.t3.micro", + id = "bananas2", class = "db.t3.micro", security_group_ids = list("sg-0ade14818d03997a4"), BackupRetentionPeriod = 0, wait = FALSE, diff --git a/tests/testthat/test-files.R b/tests/testthat/test-files.R new file mode 100644 index 0000000..ec37440 --- /dev/null +++ b/tests/testthat/test-files.R @@ -0,0 +1,37 @@ +invisible(env64$s3 <- set_s3_interface("minio")) +buckets_empty() + +demo_rds_file <- file.path(system.file(), "Meta/demo.rds") +links_file <- file.path(system.file(), "Meta/links.rds") + +test_that("aws_file_upload error behavior", { + # file doesn't exist + expect_error( + aws_file_upload( + "file_doesnt_exist.txt", + s3_path("s64-test-2", "file_doesnt_exist.txt") + ), + "is not TRUE" + ) +}) + +test_that("aws_file_upload - 1 file", { + res <- aws_file_upload( + demo_rds_file, + s3_path("s64-test-2", basename(demo_rds_file)), + force = TRUE + ) + + expect_type(res, "character") + expect_length(res, 1) +}) + +# test_that("aws_file_upload - many files", { +# aws_file_upload(demo_rds_file, "s3://not-a-bucket/eee.rds"), +# expect_type(res, "character") +# expect_length(res, 1) +# }) + +# cleanup +buckets_empty() +invisible(env64$s3 <- set_s3_interface("aws")) From c9685e429f066bbf77bbcda0de553769730161da Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Fri, 22 Mar 2024 22:46:19 -0700 Subject: [PATCH 02/20] check on branch minio --- .github/workflows/R-CMD-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index bdfd636..1a2488c 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -1,7 +1,7 @@ # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples on: push: - branches: [main] + branches: [main, minio] pull_request: branches: [main] From 62ef970a8e65ac1afb21371e1b2cd53c753d8f32 Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Fri, 22 Mar 2024 22:51:45 -0700 Subject: [PATCH 03/20] revert regular check flow back to normal, new minio specific flow --- .github/workflows/R-CMD-check-minio.yaml | 57 ++++++++++++++++++++++++ .github/workflows/R-CMD-check.yaml | 16 +------ 2 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/R-CMD-check-minio.yaml diff --git a/.github/workflows/R-CMD-check-minio.yaml b/.github/workflows/R-CMD-check-minio.yaml new file mode 100644 index 0000000..77b8fc6 --- /dev/null +++ b/.github/workflows/R-CMD-check-minio.yaml @@ -0,0 +1,57 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +on: + push: + branches: [minio] + +name: R-CMD-check-minio + +jobs: + R-CMD-check: + runs-on: ubuntu-latest + + name: ubuntu-r-release + + strategy: + fail-fast: false + + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + R_KEEP_PKG_SOURCE: yes + + steps: + - uses: actions/checkout@v3 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: ${{ matrix.config.http-user-agent }} + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::rcmdcheck + needs: check + + - name: Start Minio + run: | + wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb + sudo dpkg -i minio.deb + mkdir ~/minio + minio server ~/minio --console-address :9000 & + env: + MINIO_USER: ${{ secrets.MINIO_USER }} + MINIO_PWD: ${{ secrets.MINIO_PWD }} + MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} + + - uses: r-lib/actions/check-r-package@v2 + with: + upload-snapshots: true + env: + AWS_REGION: a + AWS_ACCESS_KEY_ID: b + AWS_SECRET_ACCESS_KEY: c + MINIO_USER: ${{ secrets.MINIO_USER }} + MINIO_PWD: ${{ secrets.MINIO_PWD }} + MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 1a2488c..bbea2e2 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -1,7 +1,7 @@ # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples on: push: - branches: [main, minio] + branches: [main] pull_request: branches: [main] @@ -43,17 +43,6 @@ jobs: extra-packages: any::rcmdcheck needs: check - - name: Start Minio - run: | - wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb - sudo dpkg -i minio.deb - mkdir ~/minio - minio server ~/minio --console-address :9001 - env: - MINIO_USER: ${{ secrets.MINIO_USER }} - MINIO_PWD: ${{ secrets.MINIO_PWD }} - MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} - - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true @@ -61,6 +50,3 @@ jobs: AWS_REGION: a AWS_ACCESS_KEY_ID: b AWS_SECRET_ACCESS_KEY: c - MINIO_USER: ${{ secrets.MINIO_USER }} - MINIO_PWD: ${{ secrets.MINIO_PWD }} - MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} From d7584a377be38e32dfe419b9ead841d44f68afe4 Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Sat, 23 Mar 2024 06:41:06 -0700 Subject: [PATCH 04/20] minio port 9001 --- .github/workflows/R-CMD-check-minio.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check-minio.yaml b/.github/workflows/R-CMD-check-minio.yaml index 77b8fc6..d7cdad8 100644 --- a/.github/workflows/R-CMD-check-minio.yaml +++ b/.github/workflows/R-CMD-check-minio.yaml @@ -39,7 +39,7 @@ jobs: wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb sudo dpkg -i minio.deb mkdir ~/minio - minio server ~/minio --console-address :9000 & + minio server ~/minio --console-address :9001 & env: MINIO_USER: ${{ secrets.MINIO_USER }} MINIO_PWD: ${{ secrets.MINIO_PWD }} From ba5e318767dda5c4291714713df2bc4bf90607e2 Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Sat, 23 Mar 2024 06:45:21 -0700 Subject: [PATCH 05/20] tickle actions From b493e02b92eb740d19d8300b97e7a175cc419615 Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Sat, 23 Mar 2024 07:15:58 -0700 Subject: [PATCH 06/20] tickle actions again From 912e1503ce76f5f2fb359791e82c75441b1fcf3f Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Sat, 23 Mar 2024 13:15:24 -0700 Subject: [PATCH 07/20] try minio on linux and macos, skip minio required tests on windows --- .github/workflows/R-CMD-check.yaml | 24 +++++++++++++++++++++++- tests/testthat/test-files.R | 2 ++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index bbea2e2..d284f4d 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -1,9 +1,10 @@ # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples on: push: - branches: [main] + branches: [main, minio] pull_request: branches: [main] + workflow_dispatch: name: R-CMD-check @@ -26,6 +27,9 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} R_KEEP_PKG_SOURCE: yes + MINIO_USER: ${{ secrets.MINIO_USER }} + MINIO_PWD: ${{ secrets.MINIO_PWD }} + MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} steps: - uses: actions/checkout@v3 @@ -43,6 +47,24 @@ jobs: extra-packages: any::rcmdcheck needs: check + - name: Start Minio + run: | + if runner.os == "Linux"; then + wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb + sudo dpkg -i minio.deb + mkdir ~/minio + minio server ~/minio --console-address :9001 & + elif runner.os == "macOS"; then + curl -O https://dl.min.io/server/minio/release/darwin-arm64/minio + chmod +x ./minio + sudo mv ./minio /usr/local/bin/ + export MINIO_CONFIG_ENV_FILE=/etc/default/minio + minio server --console-address :9001 & + else; + echo "not doing windows for now" + fi + shell: bash + - uses: r-lib/actions/check-r-package@v2 with: upload-snapshots: true diff --git a/tests/testthat/test-files.R b/tests/testthat/test-files.R index ec37440..505c8a3 100644 --- a/tests/testthat/test-files.R +++ b/tests/testthat/test-files.R @@ -1,3 +1,5 @@ +skip_on_os("windows") + invisible(env64$s3 <- set_s3_interface("minio")) buckets_empty() From 37cfb862a9afb40e7ddf312d815140d745c8c6d3 Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Sat, 23 Mar 2024 14:47:59 -0700 Subject: [PATCH 08/20] fix bash syntax --- .github/workflows/R-CMD-check.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index d284f4d..d7db87c 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -49,12 +49,12 @@ jobs: - name: Start Minio run: | - if runner.os == "Linux"; then + if [ '$RUNNER_OS' == 'Linux' ]; then wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb sudo dpkg -i minio.deb mkdir ~/minio minio server ~/minio --console-address :9001 & - elif runner.os == "macOS"; then + elif [ '$RUNNER_OS' == 'macOS' ]; then curl -O https://dl.min.io/server/minio/release/darwin-arm64/minio chmod +x ./minio sudo mv ./minio /usr/local/bin/ From 338026907b2702d6e1964e893e6b9bb2d0a4ba0d Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Sat, 23 Mar 2024 14:53:34 -0700 Subject: [PATCH 09/20] change minio start syntax to linux only; skip windows/mac in s3 files test --- .github/workflows/R-CMD-check.yaml | 20 +++++--------------- tests/testthat/test-files.R | 2 +- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index d7db87c..7e78e23 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -48,22 +48,12 @@ jobs: needs: check - name: Start Minio + if: runner.os == 'Linux' run: | - if [ '$RUNNER_OS' == 'Linux' ]; then - wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb - sudo dpkg -i minio.deb - mkdir ~/minio - minio server ~/minio --console-address :9001 & - elif [ '$RUNNER_OS' == 'macOS' ]; then - curl -O https://dl.min.io/server/minio/release/darwin-arm64/minio - chmod +x ./minio - sudo mv ./minio /usr/local/bin/ - export MINIO_CONFIG_ENV_FILE=/etc/default/minio - minio server --console-address :9001 & - else; - echo "not doing windows for now" - fi - shell: bash + wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb + sudo dpkg -i minio.deb + mkdir ~/minio + minio server ~/minio --console-address :9001 & - uses: r-lib/actions/check-r-package@v2 with: diff --git a/tests/testthat/test-files.R b/tests/testthat/test-files.R index 505c8a3..76fc6c0 100644 --- a/tests/testthat/test-files.R +++ b/tests/testthat/test-files.R @@ -1,4 +1,4 @@ -skip_on_os("windows") +skip_on_os(c("windows", "mac")) invisible(env64$s3 <- set_s3_interface("minio")) buckets_empty() From ecf7896d1519640adb4fac21dafb43c7608064e9 Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Sat, 23 Mar 2024 20:47:42 -0700 Subject: [PATCH 10/20] change from skip on os to just skip if minio not avail - should prevent skipping on dev laptops causing confusion --- tests/testthat/helper-minio.R | 5 +++++ tests/testthat/test-files.R | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/testthat/helper-minio.R b/tests/testthat/helper-minio.R index e0426f8..fd4a598 100644 --- a/tests/testthat/helper-minio.R +++ b/tests/testthat/helper-minio.R @@ -14,3 +14,8 @@ buckets_empty <- function() { invisible(purrr::map(buckets$bucket_name, bucket_delete, force = TRUE)) } } + +minio_available <- function() { + curl_check <- purrr::safely(curl::curl_fetch_memory, FALSE) + is.null(curl_check("http://127.0.0.1:9000")$error) +} diff --git a/tests/testthat/test-files.R b/tests/testthat/test-files.R index 76fc6c0..df1c14c 100644 --- a/tests/testthat/test-files.R +++ b/tests/testthat/test-files.R @@ -1,4 +1,5 @@ -skip_on_os(c("windows", "mac")) +# skip_on_os(c("windows", "mac")) +skip_if_not(minio_available(), "Minio Not Available") invisible(env64$s3 <- set_s3_interface("minio")) buckets_empty() From 3b4da912d43ece22f0487571b03db429d4783a7c Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 10:54:06 -0700 Subject: [PATCH 11/20] start minio in a tmp dir so it does not pollute pkg dir --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 30348db..90e6869 100644 --- a/Makefile +++ b/Makefile @@ -70,4 +70,4 @@ minio_start: MINIO_USER=${MINIO_USER} ;\ MINIO_PWD=${MINIO_PWD} ;\ MINIO_ENDPOINT=${MINIO_ENDPOINT} ;\ - minio server start --console-address :9090 + minio server /tmp/minio --console-address :9090 From a2471e63b442dc61d7c0b2f06f3fd6adf423419c Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 10:55:03 -0700 Subject: [PATCH 12/20] aws_file_delete - use paws instead of s3fs as its file delete is broken; use cli_abort instead of rlang abort; add files tests --- .gitignore | 18 +++ R/files.R | 28 +++- R/utils.R | 4 +- tests/testthat/test-files.R | 246 +++++++++++++++++++++++++++++++++++- 4 files changed, 284 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 9176a7a..e47cd65 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,24 @@ inst/doc tests/fixtures/aws_db_rds_create.yml tests/fixtures/aws_db_redshift_create.yml tests/fixtures/aws_secret*.yml +tests/fixtures/aws_bucket_add_user.yml +tests/fixtures/aws_bucket_add_user_setup_bucket.yml +tests/fixtures/aws_bucket_add_user_setup_user.yml +tests/fixtures/aws_bucket_get_permissions.yml +tests/fixtures/aws_bucket_get_permissions_setup_bucket.yml +tests/fixtures/aws_bucket_get_permissions_setup_users.yml +tests/fixtures/aws_bucket_permissions.yml +tests/fixtures/aws_bucket_permissions_setup_bucket.yml +tests/fixtures/aws_bucket_permissions_setup_users.yml +tests/fixtures/aws_bucket_remove_user.yml +tests/fixtures/aws_bucket_remove_user_setup_add_user.yml +tests/fixtures/aws_bucket_remove_user_setup_bucket.yml +tests/fixtures/aws_bucket_remove_user_setup_user.yml +tests/fixtures/aws_policy_attach_setup_users.yml +tests/fixtures/aws_policy_detach_setup_attach.yml +tests/fixtures/aws_policy_detach_setup_user.yml +tests/fixtures/aws_policy_list_entities_empty.yml +tests/fixtures/aws_policy_list_entities_non_empty.yml # minio start/ diff --git a/R/files.R b/R/files.R index 5870106..42e6c8a 100644 --- a/R/files.R +++ b/R/files.R @@ -64,11 +64,11 @@ aws_file_upload <- function(path, remote_path, force = FALSE, ...) { #' Download a file #' #' @export +#' @importFrom cli cli_abort #' @param remote_path (character) one or more remote S3 paths. required #' @param path (character) one or more file paths to write to. required #' @param ... named parameters passed on to [s3fs::s3_file_download()] #' @return (character) a vector of local file paths -#' @note USES A FORK OF s3fs FOR A MINOR FIX THAT MAKES LENGTH>1 INPUTS WORK #' @family files #' @examples \dontrun{ #' tfile <- tempfile() @@ -94,7 +94,18 @@ aws_file_upload <- function(path, remote_path, force = FALSE, ...) { #' } aws_file_download <- function(remote_path, path, ...) { equal_lengths(remote_path, path) - s3fs::s3_file_download(remote_path, path, ...) + res <- tryCatch( + s3fs::s3_file_download(remote_path, path), + error = function(e) e + ) + if (rlang::is_error(res)) { + if (grepl("SerializationError", res$message)) { + cli::cli_abort(c("Remote file not found", "S3 error: {res$message}")) + } else { + cli::cli_abort(res$message) + } + } + res } #' Delete a file @@ -118,7 +129,16 @@ aws_file_download <- function(remote_path, path, ...) { #' aws_file_delete(s3_path("s64-test-2", "TESTING123")) #' } aws_file_delete <- function(remote_path, ...) { - s3fs::s3_file_delete(remote_path, ...) + # FIXME: this s3fs fxn not working for some reason, not sure why yet + # using paws for now + # s3fs::s3_file_delete(remote_path, ...) + path_parsed <- path_s3_parse(remote_path) + key <- if (nchar(path_parsed[[1]]$path)) { + file.path(path_parsed[[1]]$path, path_parsed[[1]]$file) + } else { + path_parsed[[1]]$file + } + env64$s3$delete_object(path_parsed[[1]]$bucket, key) } #' File attributes @@ -217,6 +237,8 @@ aws_file_rename <- function(remote_path, new_remote_path, ...) { #' aws_file_copy(paths, "s64-test-4") #' } aws_file_copy <- function(remote_path, bucket, force = FALSE, ...) { + stop_if(rlang::is_missing(remote_path), "{.strong remote_path} is required") + stop_if(rlang::is_missing(bucket), "{.strong bucket} is required") bucket_create_if_not(bucket, force) parsed <- path_s3_parse(remote_path) parsed <- purrr::map(parsed, function(x) { diff --git a/R/utils.R b/R/utils.R index 68e45b0..43f90bb 100644 --- a/R/utils.R +++ b/R/utils.R @@ -222,8 +222,8 @@ is_class <- function(x, class) { } stop_if_not <- function(cond, msg) { - if (!cond) rlang::abort(msg) + if (!cond) cli::cli_abort(msg) } stop_if <- function(cond, msg) { - if (cond) rlang::abort(msg) + if (cond) cli::cli_abort(msg) } diff --git a/tests/testthat/test-files.R b/tests/testthat/test-files.R index df1c14c..b5938f5 100644 --- a/tests/testthat/test-files.R +++ b/tests/testthat/test-files.R @@ -1,13 +1,20 @@ -# skip_on_os(c("windows", "mac")) skip_if_not(minio_available(), "Minio Not Available") invisible(env64$s3 <- set_s3_interface("minio")) +s3fs::s3_file_system( + aws_access_key_id = "minioadmin", + aws_secret_access_key = "minioadmin", + endpoint = "http://localhost:9000" +) buckets_empty() demo_rds_file <- file.path(system.file(), "Meta/demo.rds") links_file <- file.path(system.file(), "Meta/links.rds") -test_that("aws_file_upload error behavior", { +test_that("aws_file_upload - error behavior", { + expect_error(aws_file_upload()) + expect_error(aws_file_upload("")) + # file doesn't exist expect_error( aws_file_upload( @@ -29,11 +36,236 @@ test_that("aws_file_upload - 1 file", { expect_length(res, 1) }) -# test_that("aws_file_upload - many files", { -# aws_file_upload(demo_rds_file, "s3://not-a-bucket/eee.rds"), -# expect_type(res, "character") -# expect_length(res, 1) -# }) +test_that("aws_file_upload - many files", { + aws_bucket_create("upload") + + the_files <- replicate(50, tempfile(fileext = ".txt")) + for (f in the_files) cat(letters, file = f) + + res <- aws_file_upload( + the_files, s3_path("upload", basename(the_files))) + + expect_length(res, length(the_files)) + for (f in res) expect_type(f, "character") + for (f in res) expect_match(f, "upload") + for (f in res) expect_match(f, ".txt") + + bucket_delete("upload", force = TRUE) +}) + +test_that("aws_file_download - error behavior", { + expect_error(aws_file_download()) + expect_error(aws_file_download("")) + + aws_bucket_create("download") + + # remote file doesn't exist + expect_error( + aws_file_download( + s3_path("download", "file_does_not_exist.txt"), + tempfile("file_does_not_exist", fileext=".txt") + ), + "Remote file not found" + ) + + aws_file_upload( + demo_rds_file, + s3_path("download", basename(demo_rds_file)) + ) + expect_error( + aws_file_download( + s3_path("download", basename(demo_rds_file)), + 5L + ), + "invalid 'file' argument" + ) + + bucket_delete("download", force = TRUE) +}) + +test_that("aws_file_download - many files", { + aws_bucket_create("download") + + the_files <- replicate(50, tempfile(fileext = ".txt")) + for (f in the_files) cat(letters, "\n", file = f) + + res <- aws_file_upload( + the_files, s3_path("download", basename(the_files))) + + downloaded_files <- replicate(50, tempfile(fileext = ".txt")) + out <- aws_file_download( + s3_path("download", basename(the_files)), + downloaded_files + ) + + expect_length(out, length(the_files)) + for (f in out) expect_type(f, "character") + for (f in out) expect_match(f, ".txt") + for (f in out) { + expect_equal( + strsplit(readLines(f, warn = FALSE), " ")[[1]], + letters + ) + } + + bucket_delete("download", force = TRUE) +}) + +test_that("aws_file_delete - error behavior", { + expect_error(aws_file_delete()) + + # bucket DOES NOT exist, error raised that no bucket exists + expect_error( + aws_file_delete(s3_path("a-bucket", "TESTING123")), + "bucket does not exist" + ) + + # bucket DOES exist, no error is raised + aws_bucket_create("a-bucket") + expect_no_error(aws_file_delete(s3_path("a-bucket", "TESTING123"))) + + bucket_delete("a-bucket", force = TRUE) +}) + +test_that("aws_file_delete", { + aws_bucket_create("b-bucket") + + tfile <- tempfile(fileext=".txt") + cat("Hello World!", file = tfile) + remote_path <- s3_path("b-bucket", basename(tfile)) + aws_file_upload(tfile, remote_path) + + expect_true(aws_file_exists(remote_path)) + res <- aws_file_delete(remote_path) + expect_type(res, "list") + expect_false(aws_file_exists(remote_path)) + + bucket_delete("b-bucket", force = TRUE) +}) + + +test_that("aws_file_attr - error behavior", { + expect_error(aws_file_attr()) + + # bucket DOES NOT exist + expect_error( + aws_file_attr(s3_path("a-bucket", "TESTING123")), + "SerializationError" + ) + + # bucket DOES exist, same error is raised for the file + aws_bucket_create("attr-bucket") + expect_error( + aws_file_attr(s3_path("attr-bucket", "TESTING123")), + "SerializationError" + ) + + bucket_delete("attr-bucket", force = TRUE) +}) + +test_that("aws_file_attr", { + aws_bucket_create("attr-bucket") + + tfile <- tempfile(fileext=".txt") + cat("Hello World!", file = tfile) + remote_path <- s3_path("attr-bucket", basename(tfile)) + aws_file_upload(tfile, remote_path) + + expect_true(aws_file_exists(remote_path)) + res <- aws_file_attr(remote_path) + expect_s3_class(res, "tbl") + expect_equal(NROW(res), 1) + expect_equal(res$uri, remote_path) + + bucket_delete("attr-bucket", force = TRUE) +}) + + +test_that("aws_file_exists - error behavior", { + expect_error(aws_file_exists()) + + bucket <- random_str("bucket") + + # bucket DOES NOT exist, just FALSE + expect_false(aws_file_exists(s3_path(bucket, "TESTING123"))) + + # bucket DOES exist, also FALSE + aws_bucket_create(bucket) + expect_false(aws_file_exists(s3_path(bucket, "TESTING123"))) + + bucket_delete(bucket, force = TRUE) +}) + +test_that("aws_file_exists", { + bucket <- random_str("bucket") + aws_bucket_create(bucket) + + files <- replicate(25, tempfile(fileext=".txt")) + for (i in files) { + cat("Hello World!\n\n", file = i) + remote_path <- s3_path(bucket, basename(i)) + aws_file_upload(i, remote_path) + } + + for (z in files) { + expect_true( + aws_file_exists(s3_path(bucket, basename(z))) + ) + } + expect_false( + aws_file_exists(s3_path(bucket, "non-existent.txt")) + ) + + bucket_delete(bucket, force = TRUE) +}) + +test_that("aws_file_rename", { + expect_error(aws_file_rename()) + expect_error(aws_file_rename("")) + + bucket <- random_str("bucket") + aws_bucket_create(bucket) + + aws_file_upload(links_file, s3_path(bucket, basename(links_file))) + + expect_true(aws_file_exists(s3_path(bucket, "links.rds"))) + expect_false(aws_file_exists(s3_path(bucket, "mylinks.rds"))) + + res <- aws_file_rename( + s3_path(bucket, "links.rds"), + s3_path(bucket, "mylinks.rds") + ) + + expect_false(aws_file_exists(s3_path(bucket, "links.rds"))) + expect_true(aws_file_exists(s3_path(bucket, "mylinks.rds"))) +}) + + +test_that("aws_file_copy", { + expect_error(aws_file_copy()) + expect_error(aws_file_copy("")) + + bucket <- random_str("bucket") + aws_bucket_create(bucket) + + aws_file_upload(links_file, s3_path(bucket, basename(links_file))) + + bucket_2 <- random_str("bucket") + aws_bucket_create(bucket_2) + + expect_false(aws_file_exists(s3_path(bucket_2, "links.rds"))) + + res <- aws_file_copy( + remote_path = s3_path(bucket, "links.rds"), + bucket = bucket_2 + ) + + expect_true(aws_file_exists(s3_path(bucket_2, "links.rds"))) + expect_type(res, "character") + expect_equal(res, s3_path(bucket_2, "links.rds")) +}) + + # cleanup buckets_empty() From 1cd8b31c8cc30400011a6fb3a9962f29425959cc Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 10:55:28 -0700 Subject: [PATCH 13/20] undo gitignore changes --- .gitignore | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.gitignore b/.gitignore index e47cd65..9176a7a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,24 +10,6 @@ inst/doc tests/fixtures/aws_db_rds_create.yml tests/fixtures/aws_db_redshift_create.yml tests/fixtures/aws_secret*.yml -tests/fixtures/aws_bucket_add_user.yml -tests/fixtures/aws_bucket_add_user_setup_bucket.yml -tests/fixtures/aws_bucket_add_user_setup_user.yml -tests/fixtures/aws_bucket_get_permissions.yml -tests/fixtures/aws_bucket_get_permissions_setup_bucket.yml -tests/fixtures/aws_bucket_get_permissions_setup_users.yml -tests/fixtures/aws_bucket_permissions.yml -tests/fixtures/aws_bucket_permissions_setup_bucket.yml -tests/fixtures/aws_bucket_permissions_setup_users.yml -tests/fixtures/aws_bucket_remove_user.yml -tests/fixtures/aws_bucket_remove_user_setup_add_user.yml -tests/fixtures/aws_bucket_remove_user_setup_bucket.yml -tests/fixtures/aws_bucket_remove_user_setup_user.yml -tests/fixtures/aws_policy_attach_setup_users.yml -tests/fixtures/aws_policy_detach_setup_attach.yml -tests/fixtures/aws_policy_detach_setup_user.yml -tests/fixtures/aws_policy_list_entities_empty.yml -tests/fixtures/aws_policy_list_entities_non_empty.yml # minio start/ From dfac176761240d13d46614e48ea3b5edf812a62f Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 14:18:28 -0700 Subject: [PATCH 14/20] add tests for buckets using minio; a few changes to param checkin etc for buckets fxns --- .gitignore | 18 +++++ NAMESPACE | 1 + R/bucket.R | 34 +++++--- man/aws_bucket_create.Rd | 2 +- man/aws_bucket_exists.Rd | 9 ++- man/aws_bucket_tree.Rd | 2 +- man/aws_bucket_upload.Rd | 12 ++- man/aws_buckets.Rd | 3 + man/aws_file_download.Rd | 3 - tests/testthat/test-buckets.R | 143 ++++++++++++++++++++++++++++++++++ 10 files changed, 208 insertions(+), 19 deletions(-) create mode 100644 tests/testthat/test-buckets.R diff --git a/.gitignore b/.gitignore index 9176a7a..e47cd65 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,24 @@ inst/doc tests/fixtures/aws_db_rds_create.yml tests/fixtures/aws_db_redshift_create.yml tests/fixtures/aws_secret*.yml +tests/fixtures/aws_bucket_add_user.yml +tests/fixtures/aws_bucket_add_user_setup_bucket.yml +tests/fixtures/aws_bucket_add_user_setup_user.yml +tests/fixtures/aws_bucket_get_permissions.yml +tests/fixtures/aws_bucket_get_permissions_setup_bucket.yml +tests/fixtures/aws_bucket_get_permissions_setup_users.yml +tests/fixtures/aws_bucket_permissions.yml +tests/fixtures/aws_bucket_permissions_setup_bucket.yml +tests/fixtures/aws_bucket_permissions_setup_users.yml +tests/fixtures/aws_bucket_remove_user.yml +tests/fixtures/aws_bucket_remove_user_setup_add_user.yml +tests/fixtures/aws_bucket_remove_user_setup_bucket.yml +tests/fixtures/aws_bucket_remove_user_setup_user.yml +tests/fixtures/aws_policy_attach_setup_users.yml +tests/fixtures/aws_policy_detach_setup_attach.yml +tests/fixtures/aws_policy_detach_setup_user.yml +tests/fixtures/aws_policy_list_entities_empty.yml +tests/fixtures/aws_policy_list_entities_non_empty.yml # minio start/ diff --git a/NAMESPACE b/NAMESPACE index 9ae5841..51f2298 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -73,6 +73,7 @@ export(ip_permissions_generator) export(random_user) export(s3_path) export(set_s3_interface) +importFrom(cli,cli_abort) importFrom(cli,cli_inform) importFrom(cli,cli_progress_bar) importFrom(cli,cli_progress_update) diff --git a/R/bucket.R b/R/bucket.R index a87ef31..d07089d 100644 --- a/R/bucket.R +++ b/R/bucket.R @@ -1,10 +1,11 @@ -#' Create an S3 bucket +#' Check if an S3 bucket exists #' #' @export -#' @param bucket (character) bucket name. required +#' @param bucket (character) bucket name; must be length 1. required #' @note internally uses #' [head_bucket](https://www.paws-r-sdk.com/docs/s3_head_bucket/) #' @family buckets +#' @return a single boolean (logical) #' @examples \dontrun{ #' # exists #' aws_bucket_exists(bucket = "s64-test-2") @@ -12,6 +13,8 @@ #' aws_bucket_exists(bucket = "no-bucket") #' } aws_bucket_exists <- function(bucket) { + stop_if_not(rlang::is_character(bucket), "bucket must be character") + stop_if_not(length(bucket) == 1, "length(bucket) != 1") res <- tryCatch( { env64$s3$head_bucket(Bucket = bucket) @@ -26,7 +29,7 @@ aws_bucket_exists <- function(bucket) { #' @export #' @param bucket (character) bucket name. required #' @param ... named parameters passed on to -#' [list_objects](https://www.paws-r-sdk.com/docs/s3_create_bucket/) +#' [create_bucket](https://www.paws-r-sdk.com/docs/s3_create_bucket/) #' @note Requires the env var `AWS_REGION` #' @return the bucket path (character) #' @family buckets @@ -34,6 +37,8 @@ aws_bucket_exists <- function(bucket) { #' aws_bucket_create(bucket = "s64-test-2") #' } aws_bucket_create <- function(bucket, ...) { + stop_if_not(rlang::is_character(bucket), "bucket must be character") + stop_if_not(length(bucket) == 1, "length(bucket) != 1") env64$s3$create_bucket( Bucket = bucket, CreateBucketConfiguration = @@ -80,6 +85,8 @@ bucket_create_if_not <- function(bucket, force = FALSE) { #' aws_buckets() #' } aws_bucket_delete <- function(bucket, force = FALSE, ...) { + stop_if_not(rlang::is_character(bucket), "bucket must be character") + stop_if_not(length(bucket) == 1, "length(bucket) != 1") # TODO: add a package level option to override the prompt for adv. users if (!force) { if (yesno("Are you sure you want to delete {.strong {bucket}}?")) { @@ -118,10 +125,10 @@ aws_bucket_download <- function(bucket, dest_path, ...) { #' @export #' @importFrom fs fs_bytes #' @param path (character) local path to a directory. required -#' @param bucket (character) bucket name. required #' @param max_batch (fs_bytes) maximum batch size being uploaded with each #' multipart #' @param ... named parameters passed on to [s3fs::s3_dir_upload()] +#' @inheritParams aws_bucket_delete #' @note Requires the env var `AWS_REGION`. This function prompts you to make #' sure that you want to delete the bucket. #' @family buckets @@ -144,18 +151,24 @@ aws_bucket_download <- function(bucket, dest_path, ...) { #' aws_bucket_delete(bucket_name, force = TRUE) #' aws_bucket_exists(bucket_name) aws_bucket_upload <- function( - path, bucket, max_batch = fs::fs_bytes("100MB"), + path, bucket, max_batch = fs::fs_bytes("100MB"), force = FALSE, ...) { + stop_if(rlang::is_missing(path), "{.strong path} is required") + stop_if(rlang::is_missing(bucket), "{.strong bucket} is required") if (!aws_bucket_exists(bucket)) { - if (yesno("{.strong {bucket}} does not exist. Create it?")) { - cli::cli_inform("Exiting without uploading {.strong {basename(path)}}") - return(invisible()) + if (!force) { + if (yesno("{.strong {bucket}} does not exist. Create it?")) { + cli::cli_inform("Exiting without uploading {.strong {basename(path)}}") + return(invisible()) + } } aws_bucket_create(bucket) } s3fs::s3_dir_upload( - path = path, new_path = bucket, - max_batch = max_batch + path = path, + new_path = bucket, + max_batch = max_batch, + ... ) } @@ -199,6 +212,7 @@ aws_bucket_list_objects <- function(bucket, ...) { #' @export #' @importFrom s3fs s3_dir_info #' @inherit aws_bucket_list_objects +#' @details internally uses [s3fs::s3_dir_info()] #' @note we set `refresh=TRUE` internally to make sure we return up to date #' information about your buckets rather than what's cached locally #' @family buckets diff --git a/man/aws_bucket_create.Rd b/man/aws_bucket_create.Rd index aebe2ba..a205460 100644 --- a/man/aws_bucket_create.Rd +++ b/man/aws_bucket_create.Rd @@ -10,7 +10,7 @@ aws_bucket_create(bucket, ...) \item{bucket}{(character) bucket name. required} \item{...}{named parameters passed on to -\href{https://www.paws-r-sdk.com/docs/s3_create_bucket/}{list_objects}} +\href{https://www.paws-r-sdk.com/docs/s3_create_bucket/}{create_bucket}} } \value{ the bucket path (character) diff --git a/man/aws_bucket_exists.Rd b/man/aws_bucket_exists.Rd index dc804c3..3be1128 100644 --- a/man/aws_bucket_exists.Rd +++ b/man/aws_bucket_exists.Rd @@ -2,15 +2,18 @@ % Please edit documentation in R/bucket.R \name{aws_bucket_exists} \alias{aws_bucket_exists} -\title{Create an S3 bucket} +\title{Check if an S3 bucket exists} \usage{ aws_bucket_exists(bucket) } \arguments{ -\item{bucket}{(character) bucket name. required} +\item{bucket}{(character) bucket name; must be length 1. required} +} +\value{ +a single boolean (logical) } \description{ -Create an S3 bucket +Check if an S3 bucket exists } \note{ internally uses diff --git a/man/aws_bucket_tree.Rd b/man/aws_bucket_tree.Rd index 2d48f17..903bd75 100644 --- a/man/aws_bucket_tree.Rd +++ b/man/aws_bucket_tree.Rd @@ -7,7 +7,7 @@ aws_bucket_tree(bucket, recurse = TRUE, ...) } \arguments{ -\item{bucket}{(character) bucket name. required} +\item{bucket}{(character) bucket name; must be length 1. required} \item{recurse}{(logical) returns all AWS S3 objects in lower sub directories, default: \code{TRUE}} diff --git a/man/aws_bucket_upload.Rd b/man/aws_bucket_upload.Rd index 91ad94a..774e050 100644 --- a/man/aws_bucket_upload.Rd +++ b/man/aws_bucket_upload.Rd @@ -4,7 +4,13 @@ \alias{aws_bucket_upload} \title{Upload a folder of files to create an S3 bucket} \usage{ -aws_bucket_upload(path, bucket, max_batch = fs::fs_bytes("100MB"), ...) +aws_bucket_upload( + path, + bucket, + max_batch = fs::fs_bytes("100MB"), + force = FALSE, + ... +) } \arguments{ \item{path}{(character) local path to a directory. required} @@ -14,6 +20,10 @@ aws_bucket_upload(path, bucket, max_batch = fs::fs_bytes("100MB"), ...) \item{max_batch}{(fs_bytes) maximum batch size being uploaded with each multipart} +\item{force}{(logical) force deletion without going through the prompt. +default: \code{FALSE}. Should only be set to \code{TRUE} when required for +non-interactive use.} + \item{...}{named parameters passed on to \code{\link[s3fs:upload]{s3fs::s3_dir_upload()}}} } \description{ diff --git a/man/aws_buckets.Rd b/man/aws_buckets.Rd index e787ed0..ca59d82 100644 --- a/man/aws_buckets.Rd +++ b/man/aws_buckets.Rd @@ -26,6 +26,9 @@ is an S3 bucket, with 8 columns: \description{ List S3 buckets } +\details{ +internally uses \code{\link[s3fs:info]{s3fs::s3_dir_info()}} +} \note{ we set \code{refresh=TRUE} internally to make sure we return up to date information about your buckets rather than what's cached locally diff --git a/man/aws_file_download.Rd b/man/aws_file_download.Rd index 6082cf9..1dc1813 100644 --- a/man/aws_file_download.Rd +++ b/man/aws_file_download.Rd @@ -19,9 +19,6 @@ aws_file_download(remote_path, path, ...) \description{ Download a file } -\note{ -USES A FORK OF s3fs FOR A MINOR FIX THAT MAKES LENGTH>1 INPUTS WORK -} \examples{ \dontrun{ tfile <- tempfile() diff --git a/tests/testthat/test-buckets.R b/tests/testthat/test-buckets.R new file mode 100644 index 0000000..65a8f21 --- /dev/null +++ b/tests/testthat/test-buckets.R @@ -0,0 +1,143 @@ +skip_if_not(minio_available(), "Minio Not Available") + +invisible(env64$s3 <- set_s3_interface("minio")) +s3fs::s3_file_system( + aws_access_key_id = "minioadmin", + aws_secret_access_key = "minioadmin", + endpoint = "http://localhost:9000" +) +buckets_empty() + +demo_rds_file <- file.path(system.file(), "Meta/demo.rds") +links_file <- file.path(system.file(), "Meta/links.rds") + +test_that("aws_bucket_create", { + expect_error(aws_bucket_create()) + expect_error(aws_bucket_create(5)) + expect_error(aws_bucket_create(letters)) + + bucket <- random_str("bucket") + expect_false(aws_bucket_exists(bucket)) + aws_bucket_create(bucket) + expect_true(aws_bucket_exists(bucket)) +}) + +test_that("aws_bucket_exists", { + expect_error(aws_bucket_exists()) + expect_error(aws_bucket_exists(5)) + expect_error(aws_bucket_exists(letters)) + + bucket <- random_str("bucket") + + # bucket DOES NOT exist, gives FALSE + expect_false(aws_bucket_exists(bucket)) + + # bucket DOES exist, also FALSE + aws_bucket_create(bucket) + expect_true(aws_bucket_exists(bucket)) + + bucket_delete(bucket, force = TRUE) +}) + +test_that("aws_bucket_delete", { + expect_error(aws_bucket_delete()) + expect_error(aws_bucket_delete(5)) + expect_error(aws_bucket_delete(letters)) + + bucket <- random_str("bucket") + aws_bucket_create(bucket) + expect_true(aws_bucket_exists(bucket)) + res <- aws_bucket_delete(bucket, force = TRUE) + expect_null(res) + expect_false(aws_bucket_exists(bucket)) +}) + +test_that("aws_bucket_download", { + expect_error(aws_bucket_download()) + expect_error(aws_bucket_download("")) + + bucket <- random_str("bucket") + aws_bucket_create(bucket) + + aws_file_upload(demo_rds_file, s3_path(bucket, basename(demo_rds_file))) + aws_file_upload(links_file, s3_path(bucket, basename(links_file))) + temp_dir <- file.path(tempdir(), "tmp-bucket-369") + + expect_length(list.files(temp_dir), 0) + + aws_bucket_download(bucket = bucket, dest_path = temp_dir) + + expect_length(list.files(temp_dir), 2) + + bucket_delete(bucket, force = TRUE) +}) + +test_that("aws_bucket_upload", { + expect_error(aws_bucket_upload()) + expect_error(aws_bucket_upload("")) + + res <- aws_bucket_upload( + file.path(system.file(), "Meta"), + "metabucket", + force = TRUE + ) + + expect_equal(NROW(aws_bucket_list_objects("metabucket")), 6) + expect_type(res, "character") + + bucket_delete("metabucket", force = TRUE) +}) + +test_that("aws_bucket_list_objects", { + expect_error(aws_bucket_list_objects()) + expect_error(aws_bucket_list_objects(5)) + + bucket <- random_str("bucket") + aws_bucket_create(bucket) + ffs <- list.files(file.path(system.file(), "Meta"), full.names = TRUE) + for (f in ffs) aws_file_upload(f, s3_path(bucket, basename(f))) + + res <- aws_bucket_list_objects(bucket) + + expect_s3_class(res, "tbl") + expect_equal(NROW(res), 6) + expect_s3_class(res$size, "fs_bytes") + + bucket_delete(bucket, force = TRUE) +}) + +test_that("aws_buckets", { + for (i in replicate(100, random_str("bucket"))) aws_bucket_create(i) + + res <- aws_buckets() + + expect_s3_class(res, "tbl") + expect_gt(NROW(res), 50) + + buckets_empty() +}) + +test_that("aws_bucket_tree", { + expect_error(aws_bucket_tree()) + expect_error(aws_bucket_tree("", 5)) + + bucket <- random_str("bucket") + aws_bucket_create(bucket) + ffs <- list.files(file.path(system.file(), "Meta"), full.names = TRUE) + for (f in ffs) aws_file_upload(f, s3_path(bucket, basename(f))) + + expect_output( + res <- aws_bucket_tree(bucket), + "s3://" + ) + + expect_type(res, "character") + expect_length(res, 6) + + bucket_delete(bucket, force = TRUE) +}) + + +# cleanup +buckets_empty() +invisible(env64$s3 <- set_s3_interface("aws")) From 7316b8897e51428a78bfbaad6f63fb616bcff2cc Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 14:18:54 -0700 Subject: [PATCH 15/20] remove minio standalone gh actions file --- .github/workflows/R-CMD-check-minio.yaml | 57 ------------------------ 1 file changed, 57 deletions(-) delete mode 100644 .github/workflows/R-CMD-check-minio.yaml diff --git a/.github/workflows/R-CMD-check-minio.yaml b/.github/workflows/R-CMD-check-minio.yaml deleted file mode 100644 index d7cdad8..0000000 --- a/.github/workflows/R-CMD-check-minio.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples -on: - push: - branches: [minio] - -name: R-CMD-check-minio - -jobs: - R-CMD-check: - runs-on: ubuntu-latest - - name: ubuntu-r-release - - strategy: - fail-fast: false - - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - R_KEEP_PKG_SOURCE: yes - - steps: - - uses: actions/checkout@v3 - - - uses: r-lib/actions/setup-pandoc@v2 - - - uses: r-lib/actions/setup-r@v2 - with: - r-version: release - http-user-agent: ${{ matrix.config.http-user-agent }} - use-public-rspm: true - - - uses: r-lib/actions/setup-r-dependencies@v2 - with: - extra-packages: any::rcmdcheck - needs: check - - - name: Start Minio - run: | - wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb - sudo dpkg -i minio.deb - mkdir ~/minio - minio server ~/minio --console-address :9001 & - env: - MINIO_USER: ${{ secrets.MINIO_USER }} - MINIO_PWD: ${{ secrets.MINIO_PWD }} - MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} - - - uses: r-lib/actions/check-r-package@v2 - with: - upload-snapshots: true - env: - AWS_REGION: a - AWS_ACCESS_KEY_ID: b - AWS_SECRET_ACCESS_KEY: c - MINIO_USER: ${{ secrets.MINIO_USER }} - MINIO_PWD: ${{ secrets.MINIO_PWD }} - MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} From 56472552a71f79cfec019823aae6dbd6c5cbc2b1 Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 14:22:46 -0700 Subject: [PATCH 16/20] add test coverage workflow --- .github/workflows/test-coverage.yaml | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/test-coverage.yaml diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml new file mode 100644 index 0000000..36fafb1 --- /dev/null +++ b/.github/workflows/test-coverage.yaml @@ -0,0 +1,49 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +on: + push: + branches: [main, minio] + pull_request: + branches: [main] + +name: test-coverage + +jobs: + test-coverage: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::covr + needs: coverage + + - name: Test coverage + run: | + covr::codecov( + quiet = FALSE, + clean = FALSE, + install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") + ) + shell: Rscript {0} + + - name: Show testthat output + if: always() + run: | + ## -------------------------------------------------------------------- + find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true + shell: bash + + - name: Upload test results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: coverage-test-failures + path: ${{ runner.temp }}/package From 6d2df2c61f6bcd2daa40c1d90338838aa7c90e3a Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 14:32:52 -0700 Subject: [PATCH 17/20] tweak coverage --- .github/workflows/test-coverage.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 36fafb1..152fd83 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -12,6 +12,7 @@ jobs: runs-on: ubuntu-latest env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} steps: - uses: actions/checkout@v4 From 7ccd2add69a912cfb4cdf0a8391ee13e88f67d6e Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 14:38:13 -0700 Subject: [PATCH 18/20] fix coverage build --- .github/workflows/test-coverage.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 152fd83..d069c08 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -12,7 +12,9 @@ jobs: runs-on: ubuntu-latest env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + MINIO_USER: ${{ secrets.MINIO_USER }} + MINIO_PWD: ${{ secrets.MINIO_PWD }} + MINIO_ENDPOINT: ${{ secrets.MINIO_ENDPOINT }} steps: - uses: actions/checkout@v4 @@ -26,6 +28,13 @@ jobs: extra-packages: any::covr needs: coverage + - name: Start Minio + run: | + wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240315010719.0.0_amd64.deb -O minio.deb + sudo dpkg -i minio.deb + mkdir ~/minio + minio server ~/minio --console-address :9001 & + - name: Test coverage run: | covr::codecov( @@ -34,6 +43,10 @@ jobs: install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") ) shell: Rscript {0} + env: + AWS_REGION: a + AWS_ACCESS_KEY_ID: b + AWS_SECRET_ACCESS_KEY: c - name: Show testthat output if: always() From 3091abe7d36464e721cc181d1c3d66f0f32a2b2e Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 14:41:42 -0700 Subject: [PATCH 19/20] add codecov badge to readme; only run coverage on main and not on PRs either --- .github/workflows/test-coverage.yaml | 2 -- README.Rmd | 1 + README.md | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index d069c08..f6cbbcd 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -1,8 +1,6 @@ # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples on: push: - branches: [main, minio] - pull_request: branches: [main] name: test-coverage diff --git a/README.Rmd b/README.Rmd index 822b8f9..c89d2a0 100644 --- a/README.Rmd +++ b/README.Rmd @@ -3,6 +3,7 @@ [![Project Status: Prototype – Useable, some support, open to feedback, unstable API.](https://getwilds.org/badges/badges/prototype.svg)](https://getwilds.org/badges/#prototype) [![R-CMD-check](https://github.com/getwilds/sixtyfour/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/getwilds/sixtyfour/actions/workflows/R-CMD-check.yaml) +[![codecov](https://codecov.io/gh/getwilds/sixtyfour/graph/badge.svg?token=BMER9MWIDN)](https://codecov.io/gh/getwilds/sixtyfour) A science-focused, more humane R interface to AWS. diff --git a/README.md b/README.md index ece7a99..03e3b66 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Project Status: Prototype – Useable, some support, open to feedback, unstable API.](https://getwilds.org/badges/badges/prototype.svg)](https://getwilds.org/badges/#prototype) [![R-CMD-check](https://github.com/getwilds/sixtyfour/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/getwilds/sixtyfour/actions/workflows/R-CMD-check.yaml) +[![codecov](https://codecov.io/gh/getwilds/sixtyfour/graph/badge.svg?token=BMER9MWIDN)](https://codecov.io/gh/getwilds/sixtyfour) A science-focused, more humane R interface to AWS. From 549f0c0c94d26044143d3e652a5bc9867e2fcb76 Mon Sep 17 00:00:00 2001 From: Scott Chamberlain Date: Mon, 25 Mar 2024 15:02:39 -0700 Subject: [PATCH 20/20] add info abou minio to readme --- README.Rmd | 2 ++ README.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.Rmd b/README.Rmd index c89d2a0..e5936ff 100644 --- a/README.Rmd +++ b/README.Rmd @@ -31,6 +31,7 @@ Open an issue on our [issue tracker](https://github.com/getwilds/sixtyfour/issue - See the targets in the Makefile for various package maintenance tasks. - For scanning for secrets see the make target `scan_secrets` in the Makefile. +- We use [minio][] for testing functions in this package (start with `aws_bucket`, and `aws_file`) for interacting with S3. See the make target `minio_start` for starting minio locally. The `R-CMD-check` workflow in `.github/workflows/` includes spinning up minio for running unit tests - only on linux. If you run tests and minio is not available then tests that require it will be skipped. ## Code of Conduct @@ -39,3 +40,4 @@ Please note that the sixtyfour project is released with a [Contributor Code of C [paws]: https://www.paws-r-sdk.com/ [s3fs]: https://dyfanjones.github.io/s3fs/ +[minio]: https://min.io/ diff --git a/README.md b/README.md index 03e3b66..35331d2 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Open an issue on our [issue tracker](https://github.com/getwilds/sixtyfour/issue - See the targets in the Makefile for various package maintenance tasks. - For scanning for secrets see the make target `scan_secrets` in the Makefile. +- We use [minio][] for testing functions in this package (start with `aws_bucket`, and `aws_file`) for interacting with S3. See the make target `minio_start` for starting minio locally. The `R-CMD-check` workflow in `.github/workflows/` includes spinning up minio for running unit tests - only on linux. If you run tests and minio is not available then tests that require it will be skipped. ## Code of Conduct @@ -40,3 +41,4 @@ Please note that the sixtyfour project is released with a [Contributor Code of C [paws]: https://www.paws-r-sdk.com/ [s3fs]: https://dyfanjones.github.io/s3fs/ +[minio]: https://min.io/