diff --git a/DESCRIPTION b/DESCRIPTION index 101370b09..206cc3c23 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: sits Type: Package -Version: 0.16.0 +Version: 0.16.0-1 Title: Satellite Image Time Series Analysis for Remote Sensing Data Cubes Authors@R: c(person('Rolf', 'Simoes', role = c('aut'), email = 'rolf.simoes@inpe.br'), person('Gilberto', 'Camara', role = c('aut', 'cre'), email = 'gilberto.camara@inpe.br'), diff --git a/NEWS.md b/NEWS.md index c7cdfd575..e2d2fb4c6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,9 @@ We are preparing to release the package on CRAN and are making relevant changes # What's new in SITS version 0.16.0 +### New features in SITS version 0.16.0-1 +* hotfix `sits_cube()` function to tolerate malformed paths from STAC service; + ### New features in SITS version 0.16.0 * Include `sits_apply()` function to generate new bands from existing ones; * Improve `sits_accuracy()` function to work with multiple cubes; diff --git a/R/sits_check.R b/R/sits_check.R index f7147ca50..17c374e35 100644 --- a/R/sits_check.R +++ b/R/sits_check.R @@ -869,8 +869,9 @@ NULL msg = "invalid file extension") existing_files <- file.exists(x) + existing_dirs <- dir.exists(x) .check_that( - all(existing_files), + all(existing_files | existing_dirs), local_msg = paste("file does not exist:", paste0("'", x[!existing_files], "'", collapse = ", ")), diff --git a/R/sits_classify_cube.R b/R/sits_classify_cube.R index 2239d4a40..1cdb4ea22 100644 --- a/R/sits_classify_cube.R +++ b/R/sits_classify_cube.R @@ -257,7 +257,7 @@ ) # adjust nrows and ncols - r_obj <- sits:::.raster_open_rast(out_file) + r_obj <- .raster_open_rast(out_file) file_info <- .file_info(probs_cube) file_info$nrows <- nrow(r_obj) file_info$ncols <- ncol(r_obj) diff --git a/R/sits_cube_aux_functions.R b/R/sits_cube_aux_functions.R index 259c97456..5fae1b41c 100644 --- a/R/sits_cube_aux_functions.R +++ b/R/sits_cube_aux_functions.R @@ -159,7 +159,7 @@ mv <- .config_get(key = c("sources", .cube_source(cube = cube), "collections", .cube_collection(cube = cube), - "bands", band, "missing_value",), + "bands", band, "missing_value"), default = .config_get(key = "raster_cube_missing_value") ) diff --git a/R/sits_parallel.R b/R/sits_parallel.R index 59b9e06d6..8b4ec6f84 100644 --- a/R/sits_parallel.R +++ b/R/sits_parallel.R @@ -219,8 +219,7 @@ # create progress bar pb <- NULL if (progress) - pb <- utils::txtProgressBar(min = 0, max = length(x), - style = 3, width = 50) + pb <- utils::txtProgressBar(min = 0, max = length(x), style = 3) # sequential processing if (purrr::is_null(sits_env$cluster)) { diff --git a/R/sits_raster_api_terra.R b/R/sits_raster_api_terra.R index c24d43f1e..8f32e77dc 100644 --- a/R/sits_raster_api_terra.R +++ b/R/sits_raster_api_terra.R @@ -50,7 +50,7 @@ #' @export .raster_extract.terra <- function(r_obj, xy, ...) { - terra::extract(x = r_obj, y = xy, fun = NULL, cells = FALSE, ...) + terra::extract(x = r_obj, y = xy, ...) } #' @keywords internal diff --git a/R/sits_source_api_stac.R b/R/sits_source_api_stac.R index 35ec00a05..ad690e7fe 100644 --- a/R/sits_source_api_stac.R +++ b/R/sits_source_api_stac.R @@ -196,7 +196,14 @@ # TODO: implement sits_parallel_error_retry() # open band rasters - assets <- purrr::map(paths, .raster_open_rast) + assets <- tryCatch({ + purrr::map(paths, .raster_open_rast) + }, error = function(e) { + NULL + }) + + if (is.null(assets)) + return(NULL) # get asset info asset_info <- purrr::map(assets, function(asset) { @@ -281,7 +288,38 @@ }, progress = progress) # bind cube rows - cube <- dplyr::bind_rows(tiles) %>% + cube <- dplyr::bind_rows(tiles) + + # post-condition + .check_num(nrow(cube), min = 1, msg = "number metadata rows is empty") + + # review known malformed paths + review_date <- .config_get(c("sources", source, "collections", + collection, "review_dates"), default = NA) + + if (!is.na(review_date)) { + data <- dplyr::filter(cube, date == !!review_date) %>% + tidyr::nest(assets = -tile) + + # test paths by open files... + val <- .sits_parallel_map(seq_len(nrow(data)), function(i) { + tryCatch({ + lapply(data$assets[[i]]$path, .raster_open_rast) + TRUE + }, error = function(e) FALSE) + }, progress = FALSE) + + # which tiles have passed on check + passed_tiles <- data$tile[unlist(val)] + + # exclude features by date but passed tiles + cube <- dplyr::filter( + cube, date != !!review_date | + tile %in% !!passed_tiles) + } + + # prepare cube + cube <- cube %>% tidyr::nest(file_info = -dplyr::matches(c("tile", "crs"))) %>% slider::slide_dfr(function(tile) { diff --git a/inst/extdata/config.yml b/inst/extdata/config.yml index 692318d3d..775fcfe4c 100644 --- a/inst/extdata/config.yml +++ b/inst/extdata/config.yml @@ -673,9 +673,10 @@ sources: AWS_S3_ENDPOINT : "s3.amazonaws.com" open_data: true open_data_token: false - gdalcubes_support : true + gdalcubes_support: true gdalcubes_type_format: "int16" - metadata_search : "tile" + metadata_search: "tile" + review_dates: ["2020-08-16"] DEAFRICA : s3_class : ["deafrica_cube", "stac_cube", "raster_cube"] service : "STAC" diff --git a/inst/extdata/config_internals.yml b/inst/extdata/config_internals.yml index a7d62a461..3520b955a 100644 --- a/inst/extdata/config_internals.yml +++ b/inst/extdata/config_internals.yml @@ -55,10 +55,10 @@ probs_cube_data_type : "INT2U" class_cube_data_type : "INT1U" # simultaneous connections for gdalcubes database creation -gdalcubes_open_connections : 4 +gdalcubes_open_connections : 8 # minimum number of files for gdalcubes to be run on parallel -gdalcubes_min_files_for_parallel : 10 +gdalcubes_min_files_for_parallel : 8 # configuration for gdalcubes chunk gdalcubes_chunk_size : [1, 2048, 2048] diff --git a/tests/testthat/test-config.R b/tests/testthat/test-config.R index 429e01863..801263360 100644 --- a/tests/testthat/test-config.R +++ b/tests/testthat/test-config.R @@ -26,7 +26,7 @@ test_that("User functions", { expect_equal( .config_rstac_limit(), - 1000 + 400 ) expect_equal(