Unit and Integration Testing Checklist

[kyle-messier]

beethoven unit and integration testing

[kyle-messier]

@mitchellmanware @kyle-messier

Types of tests

1. Model specifics e.g. cross-validation indices, covariate- size, name, data type, etc.
2. Package dependencies e.g. rlang injections, lightgbm installation
3. Data size, format,
4. Edge cases e.g. NA, zeros,

[kyle-messier]

chopin comments on testing that will be help ful
ropensci/software-review#638 (comment)

[mitchellmanware]

For most functions, minimum test requirement includes:

1. Function run wrapped in testthat::expect_no_error (store output in R object)

• Helps ensure general error-less run

2. Expected class of output

• testthat::expect_vector
• testthat::expect_s3_object
• testthat::expect_s4_object
• testthat::expect_true(methods::is(...))

3. Expected size of function output

• testthat::expect_length()
• testthat::expect_equal(dim(...), c(x, y, z))
• testthat::expect_true(all(length(x), c(2, 4))

4. Expected error cases with testthat::expect_error

Example tests with loadargs function

> ################################################################################
> ##### loadargs
> testthat::test_that("load_args (download)", {
+   # expect error with .csv
+   testthat::expect_error(
+     loadargs("error_file.csv", dataset = "hms")
+   )
+   # expect no error with .rds
+   testthat::expect_no_error(
+     args_rds <- loadargs(
+       testthat::test_path("..", "testdata", "load", "download_args.rds"),
+       dataset = "hms"
+     )
+   )
+   # expect a list
+   testthat::expect_true(is.list(args_rds))
+   # expect length 6
+   testthat::expect_length(args_rds, 6)
+ 
+   # expect no error with .qs
+   testthat::expect_no_error(
+     args_qs <- loadargs(
+       testthat::test_path("..", "testdata", "load", "download_args.qs"),
+       dataset = "hms"
+     )
+   )
+   # expect a list
+   testthat::expect_true(is.list(args_qs))
+   # expect length 6
+   testthat::expect_length(args_qs, 6)
+ 
+   # expect no error with two datasets
+   testthat::expect_no_error(
+     args_2 <- loadargs(
+       testthat::test_path("..", "testdata", "load", "download_args.qs"),
+       dataset = c("hms", "narr_monolevel")
+     )
+   )
+   # expect a list
+   testthat::expect_true(is.list(args_2))
+   # expect length 2
+   testthat::expect_length(args_2, 2)
+   # expect sub-lists of length 6 (hms) and 22 (narr_monolevel)
+   testthat::expect_true(all(c(6, 22) %in% unlist(lapply(args_2, length))))
+ })
Test passed 🎊

[mitchellmanware]

An additional example of unit-testing base learner functions. This test focuses on running an XGBoost model with folds = 5L and grid-search hyperparameter tuning. The testthat::test_that(...) section includes tests for temporal, spatial, and spatiotemporal cross validation types. In total, this type of test is repeated for each learner type (4) and combination of folds x tuning method (2 fold methods x 2 tuning methods = 4).

The example tests shows how the models are run on a small subset of data in a timely manner (11 seconds) and how certain expectations are placed on the output.

See https://github.com/NIEHS/beethoven/blob/mm-tests-0820/tests/testthat/test-base-xgb.R

> tic()
> ################################################################################
> ##### folds + grid tuning
> testthat::test_that("fit xgboost (folds + grid)", {
+   # import sample data
+   # sample inlcudes 2 months data for 3 sites
+   # subset to only 50 predictors for light weight
+   dt_long <- readRDS(
+     testthat::test_path("..", "testdata", "base", "dt_long.rds")
+   )
+ 
+ 
+   # set model
+   xgb_model <- switch_model("xgb")
+   # set grid
+   xgb_grid <- expand.grid(
+     mtry = c(300),
+     trees = seq(1000, 3000, 1000),
+     learn_rate = c(0.1)
+   )
+ 
+ 
+   # temporal
+   # warning is due to 3 metrics (rmse, rsq, mae)
+   testthat::expect_warning(
+     xgb1 <- fit_base_learner(
+       learner = "xgb",
+       dt_full = dt_long,
+       r_subsample = 0.3,
+       model = xgb_model,
+       folds = 5L,
+       cv_mode = "temporal",
+       tune_mode = "grid",
+       tune_grid_in = xgb_grid,
+       tune_grid_size = 1,
+       learn_rate = 0.1,
+       yvar = "Arithmetic.Mean",
+       xvar = seq(5, ncol(dt_long)),
+       nthreads = 1,
+       trim_resamples = FALSE,
+       return_best = TRUE
+     )
+   )
+   # expect a list
+   testthat::expect_true(is.list(xgb1))
+   # expect length 3
+   testthat::expect_length(xgb1, 3)
+   # expect sub-items are tibble data.frames
+   testthat::expect_equal(
+     unlist(lapply(1:3, function(x) methods::is(xgb1[[x]], "tbl_df"))),
+     c(TRUE, TRUE, TRUE)
+   )
+   # expect base predictions are numeric
+   testthat::expect_true(is.numeric(xgb1$base_prediction$.pred))
+   # expect base predictions have more than 1 value
+   # will be updated for SD/variance checks but hard with small sample
+   testthat::expect_true(
+     length(unique(xgb1$base_prediction$.pred)) > 1
+   )
+ 
+ 
+   # spatial
+   # warning is due to 3 metrics (rmse, rsq, mae)
+   testthat::expect_warning(
+     xgb2 <- fit_base_learner(
+       learner = "xgb",
+       dt_full = dt_long,
+       r_subsample = 0.3,
+       model = xgb_model,
+       folds = 5L,
+       cv_mode = "spatial",
+       tune_mode = "grid",
+       tune_grid_in = xgb_grid,
+       tune_grid_size = 1,
+       learn_rate = 0.1,
+       yvar = "Arithmetic.Mean",
+       xvar = seq(5, ncol(dt_long)),
+       nthreads = 1,
+       trim_resamples = FALSE,
+       return_best = TRUE
+     )
+   )
+   # expect a list
+   testthat::expect_true(is.list(xgb2))
+   # expect length 3
+   testthat::expect_length(xgb2, 3)
+   # expect sub-items are tibble data.frames
+   testthat::expect_equal(
+     unlist(lapply(1:3, function(x) methods::is(xgb2[[x]], "tbl_df"))),
+     c(TRUE, TRUE, TRUE)
+   )
+   # expect base predictions are numeric
+   testthat::expect_true(is.numeric(xgb2$base_prediction$.pred))
+   # expect base predictions have more than 1 value
+   # will be updated for SD/variance checks but hard with small sample
+   testthat::expect_true(
+     length(unique(xgb2$base_prediction$.pred)) > 1
+   )
+ 
+ 
+   # spatiotemporal
+   # warning is due to 3 metrics (rmse, rsq, mae)
+   testthat::expect_warning(
+     xgb3 <- fit_base_learner(
+       learner = "xgb",
+       dt_full = dt_long,
+       r_subsample = 0.3,
+       model = xgb_model,
+       folds = 5L,
+       cv_mode = "spatiotemporal",
+       tune_mode = "grid",
+       tune_grid_in = xgb_grid,
+       tune_grid_size = 1,
+       learn_rate = 0.1,
+       yvar = "Arithmetic.Mean",
+       xvar = seq(5, ncol(dt_long)),
+       nthreads = 1,
+       trim_resamples = FALSE,
+       return_best = TRUE
+     )
+   )
+   # expect a list
+   testthat::expect_true(is.list(xgb3))
+   # expect length 3
+   testthat::expect_length(xgb3, 3)
+   # expect sub-items are tibble data.frames
+   testthat::expect_equal(
+     unlist(lapply(1:3, function(x) methods::is(xgb3[[x]], "tbl_df"))),
+     c(TRUE, TRUE, TRUE)
+   )
+   # expect base predictions are numeric
+   testthat::expect_true(is.numeric(xgb3$base_prediction$.pred))
+   # expect base predictions have more than 1 value
+   # will be updated for SD/variance checks but hard with small sample
+   testthat::expect_true(
+     length(unique(xgb3$base_prediction$.pred)) > 1
+   )
+   
+ })
[16:51:13] WARNING: src/learner.cc:767: 
Parameters: { "device" } are not used.

[16:51:17] WARNING: src/learner.cc:767: 
Parameters: { "device" } are not used.

[16:51:21] WARNING: src/learner.cc:767: 
Parameters: { "device" } are not used.

Test passed 🎊
> toc()
11.005 sec elapsed