From df3ddbfb805c257c876b02caf2c530811c5ef5c3 Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Fri, 14 Jan 2022 23:18:47 -0300 Subject: [PATCH 01/10] fix progress bar width --- R/sits_parallel.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/R/sits_parallel.R b/R/sits_parallel.R index 59b9e06d6..8b4ec6f84 100644 --- a/R/sits_parallel.R +++ b/R/sits_parallel.R @@ -219,8 +219,7 @@ # create progress bar pb <- NULL if (progress) - pb <- utils::txtProgressBar(min = 0, max = length(x), - style = 3, width = 50) + pb <- utils::txtProgressBar(min = 0, max = length(x), style = 3) # sequential processing if (purrr::is_null(sits_env$cluster)) { From b5fcbb5fa837d42816e07e962f80408c63054e29 Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Fri, 14 Jan 2022 23:19:19 -0300 Subject: [PATCH 02/10] introduces fault tolerance in api_stac --- R/sits_source_api_stac.R | 42 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/R/sits_source_api_stac.R b/R/sits_source_api_stac.R index 35ec00a05..ad690e7fe 100644 --- a/R/sits_source_api_stac.R +++ b/R/sits_source_api_stac.R @@ -196,7 +196,14 @@ # TODO: implement sits_parallel_error_retry() # open band rasters - assets <- purrr::map(paths, .raster_open_rast) + assets <- tryCatch({ + purrr::map(paths, .raster_open_rast) + }, error = function(e) { + NULL + }) + + if (is.null(assets)) + return(NULL) # get asset info asset_info <- purrr::map(assets, function(asset) { @@ -281,7 +288,38 @@ }, progress = progress) # bind cube rows - cube <- dplyr::bind_rows(tiles) %>% + cube <- dplyr::bind_rows(tiles) + + # post-condition + .check_num(nrow(cube), min = 1, msg = "number metadata rows is empty") + + # review known malformed paths + review_date <- .config_get(c("sources", source, "collections", + collection, "review_dates"), default = NA) + + if (!is.na(review_date)) { + data <- dplyr::filter(cube, date == !!review_date) %>% + tidyr::nest(assets = -tile) + + # test paths by open files... + val <- .sits_parallel_map(seq_len(nrow(data)), function(i) { + tryCatch({ + lapply(data$assets[[i]]$path, .raster_open_rast) + TRUE + }, error = function(e) FALSE) + }, progress = FALSE) + + # which tiles have passed on check + passed_tiles <- data$tile[unlist(val)] + + # exclude features by date but passed tiles + cube <- dplyr::filter( + cube, date != !!review_date | + tile %in% !!passed_tiles) + } + + # prepare cube + cube <- cube %>% tidyr::nest(file_info = -dplyr::matches(c("tile", "crs"))) %>% slider::slide_dfr(function(tile) { From 185dfe68a4e23dd95f2bec1defcdab25f5c2c556 Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Fri, 14 Jan 2022 23:19:39 -0300 Subject: [PATCH 03/10] introduces 'review_dates' --- inst/extdata/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/inst/extdata/config.yml b/inst/extdata/config.yml index 692318d3d..775fcfe4c 100644 --- a/inst/extdata/config.yml +++ b/inst/extdata/config.yml @@ -673,9 +673,10 @@ sources: AWS_S3_ENDPOINT : "s3.amazonaws.com" open_data: true open_data_token: false - gdalcubes_support : true + gdalcubes_support: true gdalcubes_type_format: "int16" - metadata_search : "tile" + metadata_search: "tile" + review_dates: ["2020-08-16"] DEAFRICA : s3_class : ["deafrica_cube", "stac_cube", "raster_cube"] service : "STAC" From eafce9cf1422a07180edd22f49732f49126cd275 Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Fri, 14 Jan 2022 23:20:00 -0300 Subject: [PATCH 04/10] improve config gdalcubes params --- inst/extdata/config_internals.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inst/extdata/config_internals.yml b/inst/extdata/config_internals.yml index a7d62a461..3520b955a 100644 --- a/inst/extdata/config_internals.yml +++ b/inst/extdata/config_internals.yml @@ -55,10 +55,10 @@ probs_cube_data_type : "INT2U" class_cube_data_type : "INT1U" # simultaneous connections for gdalcubes database creation -gdalcubes_open_connections : 4 +gdalcubes_open_connections : 8 # minimum number of files for gdalcubes to be run on parallel -gdalcubes_min_files_for_parallel : 10 +gdalcubes_min_files_for_parallel : 8 # configuration for gdalcubes chunk gdalcubes_chunk_size : [1, 2048, 2048] From 1f16cde207a4a5e25308d6a5d3e5f8a92bab5105 Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Fri, 14 Jan 2022 23:24:16 -0300 Subject: [PATCH 05/10] update hotfix version --- DESCRIPTION | 2 +- NEWS.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 101370b09..206cc3c23 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: sits Type: Package -Version: 0.16.0 +Version: 0.16.0-1 Title: Satellite Image Time Series Analysis for Remote Sensing Data Cubes Authors@R: c(person('Rolf', 'Simoes', role = c('aut'), email = 'rolf.simoes@inpe.br'), person('Gilberto', 'Camara', role = c('aut', 'cre'), email = 'gilberto.camara@inpe.br'), diff --git a/NEWS.md b/NEWS.md index c7cdfd575..e2d2fb4c6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,9 @@ We are preparing to release the package on CRAN and are making relevant changes # What's new in SITS version 0.16.0 +### New features in SITS version 0.16.0-1 +* hotfix `sits_cube()` function to tolerate malformed paths from STAC service; + ### New features in SITS version 0.16.0 * Include `sits_apply()` function to generate new bands from existing ones; * Improve `sits_accuracy()` function to work with multiple cubes; From fe186c584c08d28d8ab00c72300f37c3a97c5279 Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Sat, 15 Jan 2022 00:08:50 -0300 Subject: [PATCH 06/10] remove ':::' --- R/sits_classify_cube.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/sits_classify_cube.R b/R/sits_classify_cube.R index 2239d4a40..1cdb4ea22 100644 --- a/R/sits_classify_cube.R +++ b/R/sits_classify_cube.R @@ -257,7 +257,7 @@ ) # adjust nrows and ncols - r_obj <- sits:::.raster_open_rast(out_file) + r_obj <- .raster_open_rast(out_file) file_info <- .file_info(probs_cube) file_info$nrows <- nrow(r_obj) file_info$ncols <- ncol(r_obj) From 39a29ee6128b66239f05c10feed5394e023ad1c7 Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Sat, 15 Jan 2022 10:51:36 -0300 Subject: [PATCH 07/10] include test for directory --- R/sits_check.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/sits_check.R b/R/sits_check.R index f7147ca50..17c374e35 100644 --- a/R/sits_check.R +++ b/R/sits_check.R @@ -869,8 +869,9 @@ NULL msg = "invalid file extension") existing_files <- file.exists(x) + existing_dirs <- dir.exists(x) .check_that( - all(existing_files), + all(existing_files | existing_dirs), local_msg = paste("file does not exist:", paste0("'", x[!existing_files], "'", collapse = ", ")), From abc12c15cf71bc1c72155cb78271d9b456ef0a6f Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Sat, 15 Jan 2022 10:51:51 -0300 Subject: [PATCH 08/10] remove empty argument --- R/sits_cube_aux_functions.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/sits_cube_aux_functions.R b/R/sits_cube_aux_functions.R index 259c97456..5fae1b41c 100644 --- a/R/sits_cube_aux_functions.R +++ b/R/sits_cube_aux_functions.R @@ -159,7 +159,7 @@ mv <- .config_get(key = c("sources", .cube_source(cube = cube), "collections", .cube_collection(cube = cube), - "bands", band, "missing_value",), + "bands", band, "missing_value"), default = .config_get(key = "raster_cube_missing_value") ) From a7d05b92ab47dd7357cc22095b925b1c44c731db Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Sat, 15 Jan 2022 10:52:07 -0300 Subject: [PATCH 09/10] update terra::extract call --- R/sits_raster_api_terra.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/sits_raster_api_terra.R b/R/sits_raster_api_terra.R index c24d43f1e..8f32e77dc 100644 --- a/R/sits_raster_api_terra.R +++ b/R/sits_raster_api_terra.R @@ -50,7 +50,7 @@ #' @export .raster_extract.terra <- function(r_obj, xy, ...) { - terra::extract(x = r_obj, y = xy, fun = NULL, cells = FALSE, ...) + terra::extract(x = r_obj, y = xy, ...) } #' @keywords internal From d9c24f4cdc926fc8f7be2d4d8923f9b021378db8 Mon Sep 17 00:00:00 2001 From: Rolf Simoes Date: Sat, 15 Jan 2022 10:52:28 -0300 Subject: [PATCH 10/10] update config test --- tests/testthat/test-config.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-config.R b/tests/testthat/test-config.R index 429e01863..801263360 100644 --- a/tests/testthat/test-config.R +++ b/tests/testthat/test-config.R @@ -26,7 +26,7 @@ test_that("User functions", { expect_equal( .config_rstac_limit(), - 1000 + 400 ) expect_equal(