From 8fb38187601b16be85539cae93d3a8fc6a8e6a03 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Thu, 25 Jan 2024 13:05:27 +0100 Subject: [PATCH 01/47] feat(Learner): support bundling property This is useful for learners that e.g. rely on external pointers and that need so processing of the model to be serializable. --- DESCRIPTION | 2 +- NEWS.md | 4 ++ R/Learner.R | 48 ++++++++++++++++++- R/benchmark.R | 5 +- R/mlr_reflections.R | 2 +- R/resample.R | 5 +- R/worker.R | 6 ++- man-roxygen/param_bundle.R | 3 ++ man-roxygen/param_learner_properties.R | 2 + man/Learner.Rd | 29 ++++++++++++ man/LearnerClassif.Rd | 4 ++ man/LearnerRegr.Rd | 4 ++ man/benchmark.Rd | 7 ++- man/mlr_learners_classif.debug.Rd | 2 + man/mlr_learners_classif.featureless.Rd | 2 + man/mlr_learners_classif.rpart.Rd | 2 + man/mlr_learners_regr.debug.Rd | 2 + man/mlr_learners_regr.featureless.Rd | 2 + man/mlr_learners_regr.rpart.Rd | 2 + man/resample.Rd | 7 ++- tests/testthat/test_Learner.R | 61 +++++++++++++++++++++++++ tests/testthat/test_benchmark.R | 39 ++++++++++++++++ tests/testthat/test_resample.R | 37 +++++++++++++++ 23 files changed, 267 insertions(+), 10 deletions(-) create mode 100644 man-roxygen/param_bundle.R diff --git a/DESCRIPTION b/DESCRIPTION index 288f93209..cdf32f34c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -74,7 +74,7 @@ Config/testthat/edition: 3 Config/testthat/parallel: false NeedsCompilation: no Roxygen: list(markdown = TRUE, r6 = TRUE) -RoxygenNote: 7.2.3 +RoxygenNote: 7.2.3.9000 Collate: 'mlr_reflections.R' 'BenchmarkResult.R' diff --git a/NEWS.md b/NEWS.md index e1274404e..e6e2cdcf7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # mlr3 (development version) +* Feat: added support for `"bundle"` property, which allows learners to process +models so they can be serialized. This happens automatically during `resample()` +and `benchmark()`. The naming was inspired by the {bundle} package. + # mlr3 0.17.2 * Skip new `data.table` tests on mac. diff --git a/R/Learner.R b/R/Learner.R index 9f1863548..a1b76a48c 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -164,6 +164,10 @@ Learner = R6Class("Learner", self$predict_types = assert_ordered_set(predict_types, names(mlr_reflections$learner_predict_types[[task_type]]), empty.ok = FALSE, .var.name = "predict_types") private$.predict_type = predict_types[1L] + + if (!is.null(private$.bundle) && !is.null(private$.unbundle)) { + properties = c("bundle", properties) + } self$properties = sort(assert_subset(properties, mlr_reflections$learner_properties[[task_type]])) self$data_formats = assert_subset(data_formats, mlr_reflections$data_formats) self$packages = union("mlr3", assert_character(packages, any.missing = FALSE, min.chars = 1L)) @@ -184,7 +188,7 @@ Learner = R6Class("Learner", #' @param ... (ignored). print = function(...) { catn(format(self), if (is.null(self$label) || is.na(self$label)) "" else paste0(": ", self$label)) - catn(str_indent("* Model:", if (is.null(self$model)) "-" else class(self$model)[1L])) + catn(str_indent("* Model:", if (is.null(self$model)) "-" else if (isTRUE(self$bundled)) "" else paste0(class(self$model)[1L]))) catn(str_indent("* Parameters:", as_short_string(self$param_set$values, 1000L))) catn(str_indent("* Packages:", self$packages)) catn(str_indent("* Predict Types: ", replace(self$predict_types, self$predict_types == self$predict_type, paste0("[", self$predict_type, "]")))) @@ -206,6 +210,38 @@ Learner = R6Class("Learner", open_help(self$man) }, + #' @description + #' Bundles the learner's model so it can be serialized and deserialized. + #' Does nothing if the learner does not support bundling. + bundle = function() { + if (is.null(self$model)) { + stopf("Cannot bundle, Learner '%s' has not been trained yet", self$id) + } + if (isTRUE(self$bundled)) { + lg$warn("Learner '%s' has already been bundled, skipping.", self$id) + } else if ("bundle" %in% self$properties) { + self$model = private$.bundle(self$model) + self$state$bundled = TRUE + } + invisible(self) + }, + + #' @description + #' Unbundles the learner's model so it can be used for prediction. + #' Does nothing if the learner does not support (un)bundling. + unbundle = function() { + if (is.null(self$model)) { + stopf("Cannot unbundle, Learner '%s' has not been trained yet", self$id) + } + if (isFALSE(self$bundled)) { + lg$warn("Learner '%s' has not been bundled, skipping.", self$id) + } else if (isTRUE(self$bundled)) { + self$model = private$.unbundle(self$model) + self$state$bundled = FALSE + } + invisible(self) + }, + #' @description #' Train the learner on a set of observations of the provided `task`. #' Mutates the learner by reference, i.e. stores the model alongside other information in field `$state`. @@ -279,6 +315,10 @@ Learner = R6Class("Learner", stopf("Cannot predict, Learner '%s' has not been trained yet", self$id) } + if (isTRUE(self$bundled)) { + stopf("Cannot predict, Learner '%s' has not been unbundled yet", self$id) + } + if (isTRUE(self$parallel_predict) && nbrOfWorkers() > 1L) { row_ids = row_ids %??% task$row_ids chunked = chunk_vector(row_ids, n_chunks = nbrOfWorkers(), shuffle = FALSE) @@ -388,6 +428,12 @@ Learner = R6Class("Learner", self$state$model }, + #' @field bundled (`logical(1)` or `NULL`)\cr + #' Indicates whether the model has been bundled (`TRUE`), unbudled (`FALSE`), or neither (`NULL`). + bundled = function(rhs) { + assert_ro_binding(rhs) + self$state$bundled + }, #' @field timings (named `numeric(2)`)\cr #' Elapsed time in seconds for the steps `"train"` and `"predict"`. diff --git a/R/benchmark.R b/R/benchmark.R index d1223a46f..cfe62d127 100644 --- a/R/benchmark.R +++ b/R/benchmark.R @@ -14,6 +14,7 @@ #' @template param_encapsulate #' @template param_allow_hotstart #' @template param_clone +#' @template param_bundle #' #' @return [BenchmarkResult]. #' @@ -77,7 +78,7 @@ #' ## Get the training set of the 2nd iteration of the featureless learner on penguins #' rr = bmr$aggregate()[learner_id == "classif.featureless"]$resample_result[[1]] #' rr$resampling$train_set(2) -benchmark = function(design, store_models = FALSE, store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling")) { +benchmark = function(design, store_models = FALSE, store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling"), bundle = TRUE) { assert_subset(clone, c("task", "learner", "resampling")) assert_data_frame(design, min.rows = 1L) assert_names(names(design), must.include = c("task", "learner", "resampling")) @@ -183,7 +184,7 @@ benchmark = function(design, store_models = FALSE, store_backends = TRUE, encaps res = future_map(n, workhorse, task = grid$task, learner = grid$learner, resampling = grid$resampling, iteration = grid$iteration, param_values = grid$param_values, mode = grid$mode, - MoreArgs = list(store_models = store_models, lgr_threshold = lgr_threshold, pb = pb) + MoreArgs = list(store_models = store_models, lgr_threshold = lgr_threshold, pb = pb, bundle = bundle) ) grid = insert_named(grid, list( diff --git a/R/mlr_reflections.R b/R/mlr_reflections.R index 7998c30f6..ebe30d99f 100644 --- a/R/mlr_reflections.R +++ b/R/mlr_reflections.R @@ -125,7 +125,7 @@ local({ ) ### Learner - tmp = c("featureless", "missings", "weights", "importance", "selected_features", "oob_error", "loglik", "hotstart_forward", "hotstart_backward") + tmp = c("featureless", "missings", "weights", "importance", "selected_features", "oob_error", "loglik", "hotstart_forward", "hotstart_backward", "bundle") mlr_reflections$learner_properties = list( classif = c(tmp, "twoclass", "multiclass"), regr = tmp diff --git a/R/resample.R b/R/resample.R index 9e3467e46..5a3efdd79 100644 --- a/R/resample.R +++ b/R/resample.R @@ -14,6 +14,7 @@ #' @template param_encapsulate #' @template param_allow_hotstart #' @template param_clone +#' @template param_bundle #' @return [ResampleResult]. #' #' @template section_predict_sets @@ -54,7 +55,7 @@ #' bmr1 = as_benchmark_result(rr) #' bmr2 = as_benchmark_result(rr_featureless) #' print(bmr1$combine(bmr2)) -resample = function(task, learner, resampling, store_models = FALSE, store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling")) { +resample = function(task, learner, resampling, store_models = FALSE, store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling"), bundle = TRUE) { assert_subset(clone, c("task", "learner", "resampling")) task = assert_task(as_task(task, clone = "task" %in% clone)) learner = assert_learner(as_learner(learner, clone = "learner" %in% clone)) @@ -111,7 +112,7 @@ resample = function(task, learner, resampling, store_models = FALSE, store_backe } res = future_map(n, workhorse, iteration = seq_len(n), learner = grid$learner, mode = grid$mode, - MoreArgs = list(task = task, resampling = resampling, store_models = store_models, lgr_threshold = lgr_threshold, pb = pb) + MoreArgs = list(task = task, resampling = resampling, store_models = store_models, lgr_threshold = lgr_threshold, pb = pb, bundle = bundle) ) data = data.table( diff --git a/R/worker.R b/R/worker.R index 8fa0008df..0baa85b87 100644 --- a/R/worker.R +++ b/R/worker.R @@ -72,6 +72,7 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL model = result$result, log = log, train_time = train_time, + bundled = if ("bundle" %in% learner$properties) FALSE else NULL, param_vals = learner$param_set$values, task_hash = task$hash, feature_names = task$feature_names, @@ -217,7 +218,7 @@ learner_predict = function(learner, task, row_ids = NULL) { } -workhorse = function(iteration, task, learner, resampling, param_values = NULL, lgr_threshold, store_models = FALSE, pb = NULL, mode = "train", is_sequential = TRUE) { +workhorse = function(iteration, task, learner, resampling, param_values = NULL, lgr_threshold, store_models = FALSE, pb = NULL, mode = "train", is_sequential = TRUE, bundle = TRUE) { if (!is.null(pb)) { pb(sprintf("%s|%s|i:%i", task$id, learner$id, iteration)) } @@ -266,6 +267,9 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, if (!store_models) { lg$debug("Erasing stored model for learner '%s'", learner$id) learner$state$model = NULL + } else if (bundle && "bundle" %in% learner$properties) { + lg$debug("Bundling model for learner '%s'", learner$id) + learner$bundle() } list(learner_state = learner$state, prediction = pdatas, param_values = learner$param_set$values, learner_hash = learner_hash) diff --git a/man-roxygen/param_bundle.R b/man-roxygen/param_bundle.R new file mode 100644 index 000000000..819d8a763 --- /dev/null +++ b/man-roxygen/param_bundle.R @@ -0,0 +1,3 @@ +#' @param bundle (`logical(1)`)\cr +#' Whether to bundle the learner(s) after the train-predict loop. +#' Default is `TRUE`. diff --git a/man-roxygen/param_learner_properties.R b/man-roxygen/param_learner_properties.R index 48f3817b1..c97d97de9 100644 --- a/man-roxygen/param_learner_properties.R +++ b/man-roxygen/param_learner_properties.R @@ -7,3 +7,5 @@ #' * `"importance"`: The learner supports extraction of importance scores, i.e. comes with an `$importance()` extractor function (see section on optional extractors in [Learner]). #' * `"selected_features"`: The learner supports extraction of the set of selected features, i.e. comes with a `$selected_features()` extractor function (see section on optional extractors in [Learner]). #' * `"oob_error"`: The learner supports extraction of estimated out of bag error, i.e. comes with a `oob_error()` extractor function (see section on optional extractors in [Learner]). +#' * `"bundle"`: To save learners with this property, you need to call `$bundle()` first. +#' If a learner is in a bundled state, you call first need to call `$unbundle()` to use it's model, e.g. for prediction. diff --git a/man/Learner.Rd b/man/Learner.Rd index ac6c02aaf..7e11f1882 100644 --- a/man/Learner.Rd +++ b/man/Learner.Rd @@ -185,6 +185,9 @@ Defaults to \code{NA}, but can be set by child classes.} \item{\code{model}}{(any)\cr The fitted model. Only available after \verb{$train()} has been called.} +\item{\code{bundled}}{(\code{logical(1)} or \code{NULL})\cr +Indicates whether the model has been bundled (\code{TRUE}), unbudled (\code{FALSE}), or neither (\code{NULL}).} + \item{\code{timings}}{(named \code{numeric(2)})\cr Elapsed time in seconds for the steps \code{"train"} and \code{"predict"}. Measured via \code{\link[mlr3misc:encapsulate]{mlr3misc::encapsulate()}}.} @@ -244,6 +247,8 @@ Stores \code{HotstartStack}.} \item \href{#method-Learner-format}{\code{Learner$format()}} \item \href{#method-Learner-print}{\code{Learner$print()}} \item \href{#method-Learner-help}{\code{Learner$help()}} +\item \href{#method-Learner-bundle}{\code{Learner$bundle()}} +\item \href{#method-Learner-unbundle}{\code{Learner$unbundle()}} \item \href{#method-Learner-train}{\code{Learner$train()}} \item \href{#method-Learner-predict}{\code{Learner$predict()}} \item \href{#method-Learner-predict_newdata}{\code{Learner$predict_newdata()}} @@ -303,6 +308,8 @@ The following properties are currently standardized and understood by learners i \item \code{"importance"}: The learner supports extraction of importance scores, i.e. comes with an \verb{$importance()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). +\item \code{"bundle"}: To save learners with this property, you need to call \verb{$bundle()} first. +If a learner is in a bundled state, you call first need to call \verb{$unbundle()} to use it's model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr @@ -367,6 +374,28 @@ Opens the corresponding help page referenced by field \verb{$man}. \if{html}{\out{
}}\preformatted{Learner$help()}\if{html}{\out{
}} } +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-Learner-bundle}{}}} +\subsection{Method \code{bundle()}}{ +Bundles the learner's model so it can be serialized and deserialized. +Does nothing if the learner does not support bundling. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{Learner$bundle()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-Learner-unbundle}{}}} +\subsection{Method \code{unbundle()}}{ +Unbundles the learner's model so it can be used for prediction. +Does nothing if the learner does not support (un)bundling. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{Learner$unbundle()}\if{html}{\out{
}} +} + } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/LearnerClassif.Rd b/man/LearnerClassif.Rd index f159744a6..7ef7b7703 100644 --- a/man/LearnerClassif.Rd +++ b/man/LearnerClassif.Rd @@ -85,6 +85,7 @@ Other Learner:
Inherited methods
}} @@ -139,6 +141,8 @@ The following properties are currently standardized and understood by learners i \item \code{"importance"}: The learner supports extraction of importance scores, i.e. comes with an \verb{$importance()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). +\item \code{"bundle"}: To save learners with this property, you need to call \verb{$bundle()} first. +If a learner is in a bundled state, you call first need to call \verb{$unbundle()} to use it's model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/LearnerRegr.Rd b/man/LearnerRegr.Rd index 5c9cbbcbf..700edf847 100644 --- a/man/LearnerRegr.Rd +++ b/man/LearnerRegr.Rd @@ -75,6 +75,7 @@ Other Learner:
Inherited methods
}} @@ -129,6 +131,8 @@ The following properties are currently standardized and understood by learners i \item \code{"importance"}: The learner supports extraction of importance scores, i.e. comes with an \verb{$importance()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). +\item \code{"bundle"}: To save learners with this property, you need to call \verb{$bundle()} first. +If a learner is in a bundled state, you call first need to call \verb{$unbundle()} to use it's model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/benchmark.Rd b/man/benchmark.Rd index 2bd1a35db..edb37a682 100644 --- a/man/benchmark.Rd +++ b/man/benchmark.Rd @@ -10,7 +10,8 @@ benchmark( store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, - clone = c("task", "learner", "resampling") + clone = c("task", "learner", "resampling"), + bundle = TRUE ) } \arguments{ @@ -57,6 +58,10 @@ Select the input objects to be cloned before proceeding by providing a set with possible values \code{"task"}, \code{"learner"} and \code{"resampling"} for \link{Task}, \link{Learner} and \link{Resampling}, respectively. Per default, all input objects are cloned.} + +\item{bundle}{(\code{logical(1)})\cr +Whether to bundle the learner(s) after the train-predict loop. +Default is \code{TRUE}.} } \value{ \link{BenchmarkResult}. diff --git a/man/mlr_learners_classif.debug.Rd b/man/mlr_learners_classif.debug.Rd index f5ca719ea..3b66b12d3 100644 --- a/man/mlr_learners_classif.debug.Rd +++ b/man/mlr_learners_classif.debug.Rd @@ -127,6 +127,7 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_classif.featureless.Rd b/man/mlr_learners_classif.featureless.Rd index 94c1b8595..53441620b 100644 --- a/man/mlr_learners_classif.featureless.Rd +++ b/man/mlr_learners_classif.featureless.Rd @@ -95,6 +95,7 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_classif.rpart.Rd b/man/mlr_learners_classif.rpart.Rd index 139dfe41c..614a73b49 100644 --- a/man/mlr_learners_classif.rpart.Rd +++ b/man/mlr_learners_classif.rpart.Rd @@ -109,6 +109,7 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_regr.debug.Rd b/man/mlr_learners_regr.debug.Rd index 66ffd60f2..e0f9a7f4d 100644 --- a/man/mlr_learners_regr.debug.Rd +++ b/man/mlr_learners_regr.debug.Rd @@ -100,6 +100,7 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_regr.featureless.Rd b/man/mlr_learners_regr.featureless.Rd index f06fcf4d6..20e840b74 100644 --- a/man/mlr_learners_regr.featureless.Rd +++ b/man/mlr_learners_regr.featureless.Rd @@ -84,6 +84,7 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_regr.rpart.Rd b/man/mlr_learners_regr.rpart.Rd index 9a83fec05..04f4c8997 100644 --- a/man/mlr_learners_regr.rpart.Rd +++ b/man/mlr_learners_regr.rpart.Rd @@ -109,6 +109,7 @@ Other Learner:
Inherited methods
}} diff --git a/man/resample.Rd b/man/resample.Rd index 825f086da..935c6a97c 100644 --- a/man/resample.Rd +++ b/man/resample.Rd @@ -12,7 +12,8 @@ resample( store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, - clone = c("task", "learner", "resampling") + clone = c("task", "learner", "resampling"), + bundle = TRUE ) } \arguments{ @@ -58,6 +59,10 @@ Select the input objects to be cloned before proceeding by providing a set with possible values \code{"task"}, \code{"learner"} and \code{"resampling"} for \link{Task}, \link{Learner} and \link{Resampling}, respectively. Per default, all input objects are cloned.} + +\item{bundle}{(\code{logical(1)})\cr +Whether to bundle the learner(s) after the train-predict loop. +Default is \code{TRUE}.} } \value{ \link{ResampleResult}. diff --git a/tests/testthat/test_Learner.R b/tests/testthat/test_Learner.R index bdd5f709d..7d1c14435 100644 --- a/tests/testthat/test_Learner.R +++ b/tests/testthat/test_Learner.R @@ -324,3 +324,64 @@ test_that("Models can be replaced", { learner$model$location = 1 expect_equal(learner$model$location, 1) }) + +test_that("bundling", { + task = tsk("mtcars") + LearnerRegrTest = R6Class("LearnerRegrTest", + inherit = LearnerRegrFeatureless, + private = list( + .bundle = function(model) { + private$.tmp_model = model + "bundle" + }, + .unbundle = function(model) { + model = private$.tmp_model + private$.tmp_model = NULL + model + }, + .tmp_model = NULL + ) + ) + + # bundled property + learner = LearnerRegrTest$new() + expect_true("bundle" %in% learner$properties) + expect_true(is.null(learner$bundled)) + + # (un)bundling only possible after training + expect_error(learner$bundle(), "has not been trained") + expect_error(learner$unbundle(), "has not been trained") + + learner$train(task) + model = learner$model + expect_false(learner$bundled) + learner$bundle() + expect_true(learner$bundled) + + # cannot predict with bundled learner + expect_error(learner$predict(task), "has not been unbundled") + expect_true(identical(learner$model, "bundle")) + + # unbundling works + learner$unbundle() + # can predict after unbundling + expect_prediction(learner$predict(task)) + # model is reset + expect_equal(learner$model, model) + # bundled is set accordingly + expect_false(learner$bundled) + + # when re-training, bundled is reset + learner$predict(task) + expect_false(learner$train(task)$bundled) + + ## unbundleable learners + lrn_rpart = lrn("regr.rpart") + + # bundled is NULL + expect_true(is.null(lrn_rpart$bundled)) + expect_true(is.null(lrn_rpart$train(task)$bundled)) + # calling (un)bundle does nothing + expect_true(is.null(lrn_rpart$bundle()$bundled)) + expect_true(is.null(lrn_rpart$unbundle()$bundled)) +}) diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index 2faaa19f6..9d1841238 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -476,3 +476,42 @@ test_that("param_values in benchmark", { expect_equal(bmr$learners$learner[[1]]$param_set$values, list(xval = 0, minsplit = 12, minbucket = 2)) expect_equal(bmr$learners$learner[[2]]$param_set$values, list(xval = 0, minsplit = 12, cp = 0.1)) }) + + +test_that("bundling", { + task = tsk("mtcars") + LearnerRegrTest = R6Class("LearnerRegrTest", + inherit = LearnerRegrFeatureless, + private = list( + .bundle = function(model) { + private$.tmp_model = model + "bundle" + }, + .unbundle = function(model) { + model = private$.tmp_model + private$.tmp_model = NULL + model + }, + .tmp_model = NULL + ) + ) + learner = LearnerRegrTest$new() + resampling = rsmp("holdout")$instantiate(task) + + # Learner can be bundled during benchmark() + bmr1 = benchmark(benchmark_grid(task, learner, resampling), store_models = TRUE, bundle = TRUE) + lrn_rec = bmr1$resample_results$resample_result[[1]]$learners[[1]] + expect_true(lrn_rec$bundled) + + # learner can be unbundled after benchmark() + lrn_rec$unbundle() + expect_false(lrn_rec$bundled) + + # result is the same with and without bundling + bmr2 = benchmark(benchmark_grid(task, lrn("regr.featureless"), resampling), store_models = TRUE, bundle = TRUE) + expect_equal(as.data.table(bmr1$aggregate())$regr.mse, as.data.table(bmr2$aggregate())$regr.mse) + + # bundling can be disabled + bmr3 = benchmark(benchmark_grid(task, learner, resampling), store_models = TRUE, bundle = FALSE) + expect_false(bmr3$resample_results$resample_result[[1]]$learners[[1]]$bundled) +}) diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index 91cefa4f7..48c38f87f 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -156,3 +156,40 @@ test_that("as_resample_result works for result data", { rr2 = as_resample_result(result_data) expect_class(rr2, "ResampleResult") }) + +test_that("bundling", { + task = tsk("mtcars") + LearnerRegrTest = R6Class("LearnerRegrTest", + inherit = LearnerRegrFeatureless, + private = list( + .bundle = function(model) { + private$.tmp_model = model + "bundle" + }, + .unbundle = function(model) { + model = private$.tmp_model + private$.tmp_model = NULL + model + }, + .tmp_model = NULL + ) + ) + learner = LearnerRegrTest$new() + + # allow to bundle during resample() + resampling = rsmp("holdout")$instantiate(task) + rr1 = resample(task, learner, resampling, store_models = TRUE, bundle = TRUE) + lrn_rec = rr1$learners[[1L]] + expect_true(lrn_rec$bundled) + expect_false(lrn_rec$unbundle()$bundled) + + # bundled resamples are equivalent to unbundled results + rr2 = resample(task, lrn("regr.featureless"), resampling, store_models = TRUE) + rr3 = resample(task, learner, resampling, store_models = FALSE) + expect_equal(rr1$aggregate(), rr3$aggregate()) + expect_equal(rr1$aggregate(), rr2$aggregate()) + + # bundling can be disabled + rr4 = resample(task, learner, resampling, bundle = FALSE, store_models = TRUE) + expect_false(rr4$learners[[1]]$bundled) +}) From 851e64c56932577910789cef3be19cf72b6e2afc Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Thu, 25 Jan 2024 14:33:49 +0100 Subject: [PATCH 02/47] fix(bundle): always (un)bundle for callr encapsulation --- R/worker.R | 15 ++++++++++++++- tests/testthat/test_Learner.R | 26 +++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/R/worker.R b/R/worker.R index 0baa85b87..61065b2e9 100644 --- a/R/worker.R +++ b/R/worker.R @@ -18,6 +18,10 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL stopf("Learner '%s' on task '%s' returned NULL during internal %s()", learner$id, task$id, mode) } + if ("bundle" %in% learner$properties && identical(learner$encapsulate[["train"]], "callr")) { + model = get_private(learner)$.bundle(model) + } + model } @@ -68,11 +72,16 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL log = append_log(NULL, "train", result$log$class, result$log$msg) train_time = result$elapsed + # callr encapsualtion causes dangling pointers between train and predict + bundled = if ("bundle" %in% learner$properties) { + identical(learner$encapsulate[["train"]], "callr") + } + learner$state = insert_named(learner$state, list( model = result$result, log = log, train_time = train_time, - bundled = if ("bundle" %in% learner$properties) FALSE else NULL, + bundled = bundled, param_vals = learner$param_set$values, task_hash = task$hash, feature_names = task$feature_names, @@ -102,6 +111,10 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL fb$id, learner = fb$clone()) } + if (isTRUE(bundled)) { + learner$unbundle() + } + learner } diff --git a/tests/testthat/test_Learner.R b/tests/testthat/test_Learner.R index 7d1c14435..dd818884e 100644 --- a/tests/testthat/test_Learner.R +++ b/tests/testthat/test_Learner.R @@ -329,6 +329,12 @@ test_that("bundling", { task = tsk("mtcars") LearnerRegrTest = R6Class("LearnerRegrTest", inherit = LearnerRegrFeatureless, + public = list( + initialize = function() { + super$initialize() + self$id = "regr.test" + } + ), private = list( .bundle = function(model) { private$.tmp_model = model @@ -337,9 +343,11 @@ test_that("bundling", { .unbundle = function(model) { model = private$.tmp_model private$.tmp_model = NULL + private$.counter = private$.counter + 1 model }, - .tmp_model = NULL + .tmp_model = NULL, + .counter = 0 ) ) @@ -384,4 +392,20 @@ test_that("bundling", { # calling (un)bundle does nothing expect_true(is.null(lrn_rpart$bundle()$bundled)) expect_true(is.null(lrn_rpart$unbundle()$bundled)) + + # callr encapsulation causes bundling + learner2 = LearnerRegrTest$new() + learner2$encapsulate = c(train = "callr") + learner2$train(task) + expect_false(learner2$bundled) + + learner3 = LearnerRegrTest$new() + learner3$encapsulate = c(train = "try") + learner3$train(task) + expect_false(learner3$bundled) + + # for callr, we had to unbundle + expect_equal(get_private(learner2)$.counter, 1) + # for other encapsulation, no need to unbundle becausse it was not bundled + expect_equal(get_private(learner3)$.counter, 0) }) From 1bcc0c7df9f11bf651ef4135a3da490ab6de1175 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Fri, 26 Jan 2024 10:16:40 +0100 Subject: [PATCH 03/47] fix: bundle cannot be present twice in properties --- R/Learner.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/Learner.R b/R/Learner.R index a1b76a48c..d6a3f332e 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -166,7 +166,7 @@ Learner = R6Class("Learner", private$.predict_type = predict_types[1L] if (!is.null(private$.bundle) && !is.null(private$.unbundle)) { - properties = c("bundle", properties) + properties = unique(c("bundle", properties)) } self$properties = sort(assert_subset(properties, mlr_reflections$learner_properties[[task_type]])) self$data_formats = assert_subset(data_formats, mlr_reflections$data_formats) From bfb568fe969b5bbdf5198e6411f0ff88bfc6cf37 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 30 Jan 2024 18:24:15 +0100 Subject: [PATCH 04/47] typo --- R/Learner.R | 2 +- man/Learner.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/Learner.R b/R/Learner.R index d6a3f332e..8d9ff1f6d 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -429,7 +429,7 @@ Learner = R6Class("Learner", }, #' @field bundled (`logical(1)` or `NULL`)\cr - #' Indicates whether the model has been bundled (`TRUE`), unbudled (`FALSE`), or neither (`NULL`). + #' Indicates whether the model has been bundled (`TRUE`), unbundled (`FALSE`), or neither (`NULL`). bundled = function(rhs) { assert_ro_binding(rhs) self$state$bundled diff --git a/man/Learner.Rd b/man/Learner.Rd index 7e11f1882..2ece82cf4 100644 --- a/man/Learner.Rd +++ b/man/Learner.Rd @@ -186,7 +186,7 @@ Defaults to \code{NA}, but can be set by child classes.} The fitted model. Only available after \verb{$train()} has been called.} \item{\code{bundled}}{(\code{logical(1)} or \code{NULL})\cr -Indicates whether the model has been bundled (\code{TRUE}), unbudled (\code{FALSE}), or neither (\code{NULL}).} +Indicates whether the model has been bundled (\code{TRUE}), unbundled (\code{FALSE}), or neither (\code{NULL}).} \item{\code{timings}}{(named \code{numeric(2)})\cr Elapsed time in seconds for the steps \code{"train"} and \code{"predict"}. From 606a5a9aa27095d7e93819223b900a69c492cd94 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 31 Jan 2024 19:25:23 +0100 Subject: [PATCH 05/47] refactor bundling --- DESCRIPTION | 1 + NAMESPACE | 3 + R/BenchmarkResult.R | 10 +++ R/Learner.R | 44 +------------ R/ResampleResult.R | 11 ++++ R/ResultData.R | 33 ++++++++++ R/bundle.R | 73 +++++++++++++++++++++ inst/testthat/helper_expectations.R | 54 ++++++++++++++++ man/BenchmarkResult.Rd | 22 +++++++ man/Learner.Rd | 27 -------- man/LearnerClassif.Rd | 2 - man/LearnerRegr.Rd | 2 - man/ResampleResult.Rd | 22 +++++++ man/ResultData.Rd | 22 +++++++ man/bundling.Rd | 58 +++++++++++++++++ man/mlr_learners_classif.debug.Rd | 2 - man/mlr_learners_classif.featureless.Rd | 2 - man/mlr_learners_classif.rpart.Rd | 2 - man/mlr_learners_regr.debug.Rd | 2 - man/mlr_learners_regr.featureless.Rd | 2 - man/mlr_learners_regr.rpart.Rd | 2 - tests/testthat/test_Learner.R | 85 ------------------------- tests/testthat/test_benchmark.R | 60 +++++++++++++++-- tests/testthat/test_bundle.R | 83 ++++++++++++++++++++++++ tests/testthat/test_resample.R | 43 +++++++++++-- 25 files changed, 485 insertions(+), 182 deletions(-) create mode 100644 R/bundle.R create mode 100644 man/bundling.Rd create mode 100644 tests/testthat/test_bundle.R diff --git a/DESCRIPTION b/DESCRIPTION index cdf32f34c..9793e8490 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -175,6 +175,7 @@ Collate: 'benchmark.R' 'benchmark_grid.R' 'bibentries.R' + 'bundle.R' 'default_measures.R' 'fix_factor_levels.R' 'helper.R' diff --git a/NAMESPACE b/NAMESPACE index f900058f0..e52c2bddf 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -208,6 +208,9 @@ export(extract_pkgs) export(filter_prediction_data) export(install_pkgs) export(is_missing_prediction_data) +export(learner_bundle) +export(learner_bundled) +export(learner_unbundle) export(lrn) export(lrns) export(mlr_learners) diff --git a/R/BenchmarkResult.R b/R/BenchmarkResult.R index 15d81427f..caf8cfec5 100644 --- a/R/BenchmarkResult.R +++ b/R/BenchmarkResult.R @@ -133,6 +133,16 @@ BenchmarkResult = R6Class("BenchmarkResult", invisible(self) }, + #' @description + #' Bundles all stored models. + bundle = function() { + private$.data$bundle() + }, + #' @description + #' Unbundles all stored models. + unbundle = function() { + private$.data$unbundle() + }, #' @description #' Returns a table with one row for each resampling iteration, including diff --git a/R/Learner.R b/R/Learner.R index 8d9ff1f6d..a1ae3941e 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -188,7 +188,7 @@ Learner = R6Class("Learner", #' @param ... (ignored). print = function(...) { catn(format(self), if (is.null(self$label) || is.na(self$label)) "" else paste0(": ", self$label)) - catn(str_indent("* Model:", if (is.null(self$model)) "-" else if (isTRUE(self$bundled)) "" else paste0(class(self$model)[1L]))) + catn(str_indent("* Model:", if (is.null(self$model)) "-" else if (learner_bundled(self)) "" else paste0(class(self$model)[1L]))) catn(str_indent("* Parameters:", as_short_string(self$param_set$values, 1000L))) catn(str_indent("* Packages:", self$packages)) catn(str_indent("* Predict Types: ", replace(self$predict_types, self$predict_types == self$predict_type, paste0("[", self$predict_type, "]")))) @@ -210,38 +210,6 @@ Learner = R6Class("Learner", open_help(self$man) }, - #' @description - #' Bundles the learner's model so it can be serialized and deserialized. - #' Does nothing if the learner does not support bundling. - bundle = function() { - if (is.null(self$model)) { - stopf("Cannot bundle, Learner '%s' has not been trained yet", self$id) - } - if (isTRUE(self$bundled)) { - lg$warn("Learner '%s' has already been bundled, skipping.", self$id) - } else if ("bundle" %in% self$properties) { - self$model = private$.bundle(self$model) - self$state$bundled = TRUE - } - invisible(self) - }, - - #' @description - #' Unbundles the learner's model so it can be used for prediction. - #' Does nothing if the learner does not support (un)bundling. - unbundle = function() { - if (is.null(self$model)) { - stopf("Cannot unbundle, Learner '%s' has not been trained yet", self$id) - } - if (isFALSE(self$bundled)) { - lg$warn("Learner '%s' has not been bundled, skipping.", self$id) - } else if (isTRUE(self$bundled)) { - self$model = private$.unbundle(self$model) - self$state$bundled = FALSE - } - invisible(self) - }, - #' @description #' Train the learner on a set of observations of the provided `task`. #' Mutates the learner by reference, i.e. stores the model alongside other information in field `$state`. @@ -315,7 +283,7 @@ Learner = R6Class("Learner", stopf("Cannot predict, Learner '%s' has not been trained yet", self$id) } - if (isTRUE(self$bundled)) { + if (isTRUE(self$state$bundled)) { stopf("Cannot predict, Learner '%s' has not been unbundled yet", self$id) } @@ -428,13 +396,6 @@ Learner = R6Class("Learner", self$state$model }, - #' @field bundled (`logical(1)` or `NULL`)\cr - #' Indicates whether the model has been bundled (`TRUE`), unbundled (`FALSE`), or neither (`NULL`). - bundled = function(rhs) { - assert_ro_binding(rhs) - self$state$bundled - }, - #' @field timings (named `numeric(2)`)\cr #' Elapsed time in seconds for the steps `"train"` and `"predict"`. #' Measured via [mlr3misc::encapsulate()]. @@ -587,7 +548,6 @@ Learner = R6Class("Learner", ) ) - #' @export rd_info.Learner = function(obj, ...) { x = c("", diff --git a/R/ResampleResult.R b/R/ResampleResult.R index cde828ea8..7f032c22e 100644 --- a/R/ResampleResult.R +++ b/R/ResampleResult.R @@ -222,6 +222,17 @@ ResampleResult = R6Class("ResampleResult", #' the object in its previous state. discard = function(backends = FALSE, models = FALSE) { private$.data$discard(backends = backends, models = models) + }, + + #' @description + #' Bundles all stored learner models. + bundle = function() { + private$.data$bundle() + }, + #' @description + #' Unbundles all stored learner models. + unbundle = function() { + private$.data$unbundle() } ), diff --git a/R/ResultData.R b/R/ResultData.R index 6550f0bfd..b54d8d83d 100644 --- a/R/ResultData.R +++ b/R/ResultData.R @@ -242,6 +242,39 @@ ResultData = R6Class("ResultData", invisible(self) }, + #' @description + #' Bundles all stored learner models. + bundle = function() { + learner_states = map(seq_len(nrow(self$data$fact)), function(i) { + state = self$data$fact[i, "learner_state"][[1L]][[1L]] + phash = self$data$fact[i, "learner_phash"][[1L]] + learner = self$data$learners[phash, "learner", on = "learner_phash"][[1L]][[1L]] + if (!is.null(state$model) && isFALSE(state$bundled)) { + state$model = get_private(learner)$.bundle(state$model) + state$bundled = TRUE + } + state + }) + self$data$fact$learner_state = learner_states + invisible(self) + }, + #' @description + #' Unbundles all stored learner models. + unbundle = function() { + learner_states = map(seq_len(nrow(self$data$fact)), function(i) { + state = self$data$fact[i, "learner_state"][[1L]][[1L]] + phash = self$data$fact[i, "learner_phash"][[1L]] + learner = self$data$learners[phash, "learner", on = "learner_phash"][[1L]][[1L]] + if (!is.null(state$model) && isTRUE(state$bundled)) { + state$model = get_private(learner)$.unbundle(state$model) + state$bundled = FALSE + } + state + }) + self$data$fact$learner_state = learner_states + invisible(self) + }, + #' @description #' Shrinks the object by discarding parts of the stored data. #' diff --git a/R/bundle.R b/R/bundle.R new file mode 100644 index 000000000..272b67bf4 --- /dev/null +++ b/R/bundle.R @@ -0,0 +1,73 @@ +#' @title (Un)bundle a Learner +#' +#' @name bundling +#' +#' @description +#' Bundling is the process of processing the model of a trained [`Learner`] so it an be successfully serialized and +#' deserialized. The naming is inspired from \CRANpkg{bundle}. +#' +#' The function: +#' * `learner_bundle(learner)`: +#' Replaces the learner's model with the bundled model. +#' * `learner_unbundle(learner) : +#' Replaces the learner's model with the unbundled model. +#' * `learner_bundled(learner)`: +#' returns `FALSE` if the learner is either not trained or not bundled, otherwise `TRUE`. +#' +#' All three functions are primarily intended to be used when implementing bundling for a [`Learner`]. +#' Users who want to (un)bundle a [`Learner`] can instead use the public methods `$bundle()` and `$unbundle()` or the +#' field `$bundled`, which is more in line with `mlr3`'s object oriented design. +#' +#' @section Implementing Bundling for a Learner: +#' In order to implement bundling for a [`Learner`], you need to add: +#' * the public methods `$bundle()` and `$unbundle()`, where you call `learner_bundle(self)` and +#' `learner_unbundle(self)` respectively. +#' * the active binding `$bundled`, where you simply call `learner_bundled(self)`. +#' * the private method `$.bundle(model)`, which takes in a [`Learner`]'s model and returns it in bundled form, +#' without modifying the learner's state. Must not depend on the learner's state. +#' * the private method `$.unbundle(model)`, which takes in a [`Learner`]'s bundled model and returns it in +#' unbundled form. Must not depend on the learner's state. +#' +#' To test the bundling implementation, you can use the internal test helper `expect_bundleable()`. +#' This is also run in `expect_learner()` when a task is provided. +#' +#' For a concrete example on how to implement bundling, see the `LearnerTorch` class from +#' [mlr3torch](https://github.com/mlr-org/mlr3torch). +#' +#' @param learner [`Learner`]\cr +#' The learner to bundle. +#' @keywords internal +#' @export +learner_unbundle = function(learner) { + if (is.null(learner$model)) { + stopf("Cannot unbundle, Learner '%s' has not been trained yet", learner$id) + } + if (isFALSE(learner$bundled)) { + warningf("Learner '%s' has not been bundled, skipping.", learner$id) + } else if (isTRUE(learner$bundled)) { + learner$model = get_private(learner)$.unbundle(learner$model) + learner$state$bundled = FALSE + } + invisible(learner) +} + +#' @rdname bundling +#' @export +learner_bundle = function(learner) { + if (is.null(learner$model)) { + stopf("Cannot bundle, Learner '%s' has not been trained yet", learner$id) + } + if (isTRUE(learner$bundled)) { + warningf("Learner '%s' has already been bundled, skipping.", learner$id) + } else if ("bundle" %in% learner$properties) { + learner$model = get_private(learner)$.bundle(learner$model) + learner$state$bundled = TRUE + } + invisible(learner) +} + +#' @rdname bundling +#' @export +learner_bundled = function(learner) { + isTRUE(learner$state$bundled) && !is.null(learner$model) +} diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index b04ea3869..352f22646 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -342,6 +342,7 @@ expect_task_generator = function(gen) { expect_learner = function(lrn, task = NULL, check_man = TRUE) { + checkmate::expect_r6(lrn, "Learner", cloneable = TRUE) expect_id(lrn$id) checkmate::expect_string(lrn$label, na.ok = TRUE) @@ -378,11 +379,64 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { checkmate::expect_class(task, "Task") checkmate::expect_subset(lrn$properties, mlr3::mlr_reflections$learner_properties[[task$task_type]]) testthat::expect_identical(lrn$task_type, task$task_type) + + if ("bundle" %in% lrn$properties) { + expect_bundleable(lrn, task) + } } if (!inherits(lrn, "GraphLearner") && !inherits(lrn, "AutoTuner")) { # still not in pipelines, breaking check in mlr3tuning checkmate::expect_class(lrn$base_learner(), "Learner") } + +} + +expect_bundleable = function(learner, task) { + expect_true("bundle" %in% learner$properties) + learner$state = NULL + + has_public = function(learner, x) { + exists(x, learner, inherits = FALSE) + } + has_private = function(learner, x) { + exists(x, get_private(learner), inherits = FALSE) + } + + expect_true(has_public(learner, "bundle") && test_function(learner$bundle, nargs = 0)) + expect_true(has_public(learner, "unbundle") && test_function(learner$unbundle, nargs = 0)) + expect_true(has_public(learner, "bundle")) + expect_true(has_private(learner, ".bundle") && test_function(get_private(learner)$.bundle, nargs = 1, args = "model")) + expect_true(has_private(learner, ".unbundle") && test_function(get_private(learner)$.unbundle, nargs = 1, args = "model")) + + expect_false(learner$bundled) + + # (un)bundling only possible after training + expect_error(learner$bundle(), "has not been trained") + expect_error(learner$unbundle(), "has not been trained") + + learner$train(task) + model = learner$model + expect_false(learner$bundled) + learner$bundle() + expect_true(learner$bundled) + + # cannot predict with bundled learner + expect_error(learner$predict(task), "has not been unbundled") + expect_true(identical(learner$model, "bundle")) + + # unbundling works + learner$unbundle() + # can predict after unbundling + expect_prediction(learner$predict(task)) + # model is reset + expect_equal(learner$model, model) + # bundled is set accordingly + expect_false(learner$bundled) + + # when re-training, bundled is reset + learner$predict(task) + expect_false(learner$train(task)$bundled) + } expect_resampling = function(r, task = NULL) { diff --git a/man/BenchmarkResult.Rd b/man/BenchmarkResult.Rd index f8d31d223..65d778214 100644 --- a/man/BenchmarkResult.Rd +++ b/man/BenchmarkResult.Rd @@ -141,6 +141,8 @@ Set of (unique) hashes of all included \link{ResampleResult}s.} \item \href{#method-BenchmarkResult-format}{\code{BenchmarkResult$format()}} \item \href{#method-BenchmarkResult-print}{\code{BenchmarkResult$print()}} \item \href{#method-BenchmarkResult-combine}{\code{BenchmarkResult$combine()}} +\item \href{#method-BenchmarkResult-bundle}{\code{BenchmarkResult$bundle()}} +\item \href{#method-BenchmarkResult-unbundle}{\code{BenchmarkResult$unbundle()}} \item \href{#method-BenchmarkResult-score}{\code{BenchmarkResult$score()}} \item \href{#method-BenchmarkResult-aggregate}{\code{BenchmarkResult$aggregate()}} \item \href{#method-BenchmarkResult-filter}{\code{BenchmarkResult$filter()}} @@ -229,6 +231,26 @@ Returns the object itself, but modified \strong{by reference}. You need to explicitly \verb{$clone()} the object beforehand if you want to keep the object in its previous state. } +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-BenchmarkResult-bundle}{}}} +\subsection{Method \code{bundle()}}{ +Bundles all stored models. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{BenchmarkResult$bundle()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-BenchmarkResult-unbundle}{}}} +\subsection{Method \code{unbundle()}}{ +Unbundles all stored models. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{BenchmarkResult$unbundle()}\if{html}{\out{
}} +} + } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/Learner.Rd b/man/Learner.Rd index 2ece82cf4..269bfe274 100644 --- a/man/Learner.Rd +++ b/man/Learner.Rd @@ -185,9 +185,6 @@ Defaults to \code{NA}, but can be set by child classes.} \item{\code{model}}{(any)\cr The fitted model. Only available after \verb{$train()} has been called.} -\item{\code{bundled}}{(\code{logical(1)} or \code{NULL})\cr -Indicates whether the model has been bundled (\code{TRUE}), unbundled (\code{FALSE}), or neither (\code{NULL}).} - \item{\code{timings}}{(named \code{numeric(2)})\cr Elapsed time in seconds for the steps \code{"train"} and \code{"predict"}. Measured via \code{\link[mlr3misc:encapsulate]{mlr3misc::encapsulate()}}.} @@ -247,8 +244,6 @@ Stores \code{HotstartStack}.} \item \href{#method-Learner-format}{\code{Learner$format()}} \item \href{#method-Learner-print}{\code{Learner$print()}} \item \href{#method-Learner-help}{\code{Learner$help()}} -\item \href{#method-Learner-bundle}{\code{Learner$bundle()}} -\item \href{#method-Learner-unbundle}{\code{Learner$unbundle()}} \item \href{#method-Learner-train}{\code{Learner$train()}} \item \href{#method-Learner-predict}{\code{Learner$predict()}} \item \href{#method-Learner-predict_newdata}{\code{Learner$predict_newdata()}} @@ -374,28 +369,6 @@ Opens the corresponding help page referenced by field \verb{$man}. \if{html}{\out{
}}\preformatted{Learner$help()}\if{html}{\out{
}} } -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-Learner-bundle}{}}} -\subsection{Method \code{bundle()}}{ -Bundles the learner's model so it can be serialized and deserialized. -Does nothing if the learner does not support bundling. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{Learner$bundle()}\if{html}{\out{
}} -} - -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-Learner-unbundle}{}}} -\subsection{Method \code{unbundle()}}{ -Unbundles the learner's model so it can be used for prediction. -Does nothing if the learner does not support (un)bundling. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{Learner$unbundle()}\if{html}{\out{
}} -} - } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/LearnerClassif.Rd b/man/LearnerClassif.Rd index 7ef7b7703..c8994c993 100644 --- a/man/LearnerClassif.Rd +++ b/man/LearnerClassif.Rd @@ -85,7 +85,6 @@ Other Learner:
Inherited methods
}} diff --git a/man/LearnerRegr.Rd b/man/LearnerRegr.Rd index 700edf847..759cbe094 100644 --- a/man/LearnerRegr.Rd +++ b/man/LearnerRegr.Rd @@ -75,7 +75,6 @@ Other Learner:
Inherited methods
}} diff --git a/man/ResampleResult.Rd b/man/ResampleResult.Rd index fef49fa6d..e01e08c2c 100644 --- a/man/ResampleResult.Rd +++ b/man/ResampleResult.Rd @@ -107,6 +107,8 @@ Note that there can be multiple rows per resampling iteration if multiple errors \item \href{#method-ResampleResult-aggregate}{\code{ResampleResult$aggregate()}} \item \href{#method-ResampleResult-filter}{\code{ResampleResult$filter()}} \item \href{#method-ResampleResult-discard}{\code{ResampleResult$discard()}} +\item \href{#method-ResampleResult-bundle}{\code{ResampleResult$bundle()}} +\item \href{#method-ResampleResult-unbundle}{\code{ResampleResult$unbundle()}} \item \href{#method-ResampleResult-clone}{\code{ResampleResult$clone()}} } } @@ -347,6 +349,26 @@ Returns the object itself, but modified \strong{by reference}. You need to explicitly \verb{$clone()} the object beforehand if you want to keeps the object in its previous state. } +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ResampleResult-bundle}{}}} +\subsection{Method \code{bundle()}}{ +Bundles all stored learner models. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{ResampleResult$bundle()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ResampleResult-unbundle}{}}} +\subsection{Method \code{unbundle()}}{ +Unbundles all stored learner models. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{ResampleResult$unbundle()}\if{html}{\out{
}} +} + } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/ResultData.Rd b/man/ResultData.Rd index bf72a7c01..22ed59241 100644 --- a/man/ResultData.Rd +++ b/man/ResultData.Rd @@ -50,6 +50,8 @@ Returns \code{NULL} if the \link{ResultData} is empty.} \item \href{#method-ResultData-prediction}{\code{ResultData$prediction()}} \item \href{#method-ResultData-combine}{\code{ResultData$combine()}} \item \href{#method-ResultData-sweep}{\code{ResultData$sweep()}} +\item \href{#method-ResultData-bundle}{\code{ResultData$bundle()}} +\item \href{#method-ResultData-unbundle}{\code{ResultData$unbundle()}} \item \href{#method-ResultData-discard}{\code{ResultData$discard()}} \item \href{#method-ResultData-as_data_table}{\code{ResultData$as_data_table()}} \item \href{#method-ResultData-logs}{\code{ResultData$logs()}} @@ -330,6 +332,26 @@ E.g., can be called after filtering/subsetting the fact table. \subsection{Returns}{ Modified \code{self} (invisibly). } +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ResultData-bundle}{}}} +\subsection{Method \code{bundle()}}{ +Bundles all stored learner models. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{ResultData$bundle()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ResultData-unbundle}{}}} +\subsection{Method \code{unbundle()}}{ +Unbundles all stored learner models. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{ResultData$unbundle()}\if{html}{\out{
}} +} + } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/bundling.Rd b/man/bundling.Rd new file mode 100644 index 000000000..dfc17b1a3 --- /dev/null +++ b/man/bundling.Rd @@ -0,0 +1,58 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/bundle.R +\name{bundling} +\alias{bundling} +\alias{learner_unbundle} +\alias{learner_bundle} +\alias{learner_bundled} +\title{(Un)bundle a Learner} +\usage{ +learner_unbundle(learner) + +learner_bundle(learner) + +learner_bundled(learner) +} +\arguments{ +\item{learner}{\code{\link{Learner}}\cr +The learner to bundle.} +} +\description{ +Bundling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and +deserialized. The naming is inspired from \CRANpkg{bundle}. + +The function: +\itemize{ +\item \code{learner_bundle(learner)}: +Replaces the learner's model with the bundled model. +\item `learner_unbundle(learner) : +Replaces the learner's model with the unbundled model. +\item \code{learner_bundled(learner)}: +returns \code{FALSE} if the learner is either not trained or not bundled, otherwise \code{TRUE}. +} + +All three functions are primarily intended to be used when implementing bundling for a \code{\link{Learner}}. +Users who want to (un)bundle a \code{\link{Learner}} can instead use the public methods \verb{$bundle()} and \verb{$unbundle()} or the +field \verb{$bundled}, which is more in line with \code{mlr3}'s object oriented design. +} +\section{Implementing Bundling for a Learner}{ + +In order to implement bundling for a \code{\link{Learner}}, you need to add: +\itemize{ +\item the public methods \verb{$bundle()} and \verb{$unbundle()}, where you call \code{learner_bundle(self)} and +\code{learner_unbundle(self)} respectively. +\item the active binding \verb{$bundled}, where you simply call \code{learner_bundled(self)}. +\item the private method \verb{$.bundle(model)}, which takes in a \code{\link{Learner}}'s model and returns it in bundled form, +without modifying the learner's state. Must not depend on the learner's state. +\item the private method \verb{$.unbundle(model)}, which takes in a \code{\link{Learner}}'s bundled model and returns it in +unbundled form. Must not depend on the learner's state. +} + +To test the bundling implementation, you can use the internal test helper \code{expect_bundleable()}. +This is also run in \code{expect_learner()} when a task is provided. + +For a concrete example on how to implement bundling, see the \code{LearnerTorch} class from +\href{https://github.com/mlr-org/mlr3torch}{mlr3torch}. +} + +\keyword{internal} diff --git a/man/mlr_learners_classif.debug.Rd b/man/mlr_learners_classif.debug.Rd index 3b66b12d3..f5ca719ea 100644 --- a/man/mlr_learners_classif.debug.Rd +++ b/man/mlr_learners_classif.debug.Rd @@ -127,7 +127,6 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_classif.featureless.Rd b/man/mlr_learners_classif.featureless.Rd index 53441620b..94c1b8595 100644 --- a/man/mlr_learners_classif.featureless.Rd +++ b/man/mlr_learners_classif.featureless.Rd @@ -95,7 +95,6 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_classif.rpart.Rd b/man/mlr_learners_classif.rpart.Rd index 614a73b49..139dfe41c 100644 --- a/man/mlr_learners_classif.rpart.Rd +++ b/man/mlr_learners_classif.rpart.Rd @@ -109,7 +109,6 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_regr.debug.Rd b/man/mlr_learners_regr.debug.Rd index e0f9a7f4d..66ffd60f2 100644 --- a/man/mlr_learners_regr.debug.Rd +++ b/man/mlr_learners_regr.debug.Rd @@ -100,7 +100,6 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_regr.featureless.Rd b/man/mlr_learners_regr.featureless.Rd index 20e840b74..f06fcf4d6 100644 --- a/man/mlr_learners_regr.featureless.Rd +++ b/man/mlr_learners_regr.featureless.Rd @@ -84,7 +84,6 @@ Other Learner:
Inherited methods
}} diff --git a/man/mlr_learners_regr.rpart.Rd b/man/mlr_learners_regr.rpart.Rd index 04f4c8997..9a83fec05 100644 --- a/man/mlr_learners_regr.rpart.Rd +++ b/man/mlr_learners_regr.rpart.Rd @@ -109,7 +109,6 @@ Other Learner:
Inherited methods
}} diff --git a/tests/testthat/test_Learner.R b/tests/testthat/test_Learner.R index dd818884e..bdd5f709d 100644 --- a/tests/testthat/test_Learner.R +++ b/tests/testthat/test_Learner.R @@ -324,88 +324,3 @@ test_that("Models can be replaced", { learner$model$location = 1 expect_equal(learner$model$location, 1) }) - -test_that("bundling", { - task = tsk("mtcars") - LearnerRegrTest = R6Class("LearnerRegrTest", - inherit = LearnerRegrFeatureless, - public = list( - initialize = function() { - super$initialize() - self$id = "regr.test" - } - ), - private = list( - .bundle = function(model) { - private$.tmp_model = model - "bundle" - }, - .unbundle = function(model) { - model = private$.tmp_model - private$.tmp_model = NULL - private$.counter = private$.counter + 1 - model - }, - .tmp_model = NULL, - .counter = 0 - ) - ) - - # bundled property - learner = LearnerRegrTest$new() - expect_true("bundle" %in% learner$properties) - expect_true(is.null(learner$bundled)) - - # (un)bundling only possible after training - expect_error(learner$bundle(), "has not been trained") - expect_error(learner$unbundle(), "has not been trained") - - learner$train(task) - model = learner$model - expect_false(learner$bundled) - learner$bundle() - expect_true(learner$bundled) - - # cannot predict with bundled learner - expect_error(learner$predict(task), "has not been unbundled") - expect_true(identical(learner$model, "bundle")) - - # unbundling works - learner$unbundle() - # can predict after unbundling - expect_prediction(learner$predict(task)) - # model is reset - expect_equal(learner$model, model) - # bundled is set accordingly - expect_false(learner$bundled) - - # when re-training, bundled is reset - learner$predict(task) - expect_false(learner$train(task)$bundled) - - ## unbundleable learners - lrn_rpart = lrn("regr.rpart") - - # bundled is NULL - expect_true(is.null(lrn_rpart$bundled)) - expect_true(is.null(lrn_rpart$train(task)$bundled)) - # calling (un)bundle does nothing - expect_true(is.null(lrn_rpart$bundle()$bundled)) - expect_true(is.null(lrn_rpart$unbundle()$bundled)) - - # callr encapsulation causes bundling - learner2 = LearnerRegrTest$new() - learner2$encapsulate = c(train = "callr") - learner2$train(task) - expect_false(learner2$bundled) - - learner3 = LearnerRegrTest$new() - learner3$encapsulate = c(train = "try") - learner3$train(task) - expect_false(learner3$bundled) - - # for callr, we had to unbundle - expect_equal(get_private(learner2)$.counter, 1) - # for other encapsulation, no need to unbundle becausse it was not bundled - expect_equal(get_private(learner3)$.counter, 0) -}) diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index 9d1841238..7ea86d417 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -482,17 +482,31 @@ test_that("bundling", { task = tsk("mtcars") LearnerRegrTest = R6Class("LearnerRegrTest", inherit = LearnerRegrFeatureless, + public = list( + initialize = function(class = "bundle") { + private$.class = class + super$initialize() + }, + bundle = function() { + learner_bundle(self) + }, + unbundle = function() { + learner_unbundle(self) + } + ), + active = list( + bundled = function() { + learner_bundled(self) + } + ), private = list( .bundle = function(model) { - private$.tmp_model = model - "bundle" + structure(list(model), class = private$.class) }, .unbundle = function(model) { - model = private$.tmp_model - private$.tmp_model = NULL - model + model[[1L]] }, - .tmp_model = NULL + .class = NULL ) ) learner = LearnerRegrTest$new() @@ -514,4 +528,38 @@ test_that("bundling", { # bundling can be disabled bmr3 = benchmark(benchmark_grid(task, learner, resampling), store_models = TRUE, bundle = FALSE) expect_false(bmr3$resample_results$resample_result[[1]]$learners[[1]]$bundled) + + # can (un)bundle benchmark result + # we are mixing differnt bundling methods + l1 = LearnerRegrTest$new(class = "a") + l2 = lrn("regr.rpart") + l3 = LearnerRegrTest$new(class = "b") + # to give l1 and l3 different hashes + class(l3) = c("LearenrRegrTest2", class(l3)[-1]) + expect_false(identical(l1$phash, l3$phash)) + + design = benchmark_grid(tsk("mtcars"), list(l1, l2, l3), rsmp("cv", folds = 3)) + bmr = benchmark(design, store_models = TRUE) + + bmr$unbundle() + + walk(bmr$resample_result(1)$learners, function(l) { + expect_class(l$model, "regr.featureless_model") + }) + walk(bmr$resample_result(2)$learners, function(l) { + expect_class(l$model, "rpart") + }) + walk(bmr$resample_result(3)$learners, function(l) { + expect_class(l$model, "regr.featureless_model") + }) + bmr$bundle() + walk(bmr$resample_result(1)$learners, function(l) { + expect_class(l$model, "a") + }) + walk(bmr$resample_result(2)$learners, function(l) { + expect_class(l$model, "rpart") + }) + walk(bmr$resample_result(3)$learners, function(l) { + expect_class(l$model, "b") + }) }) diff --git a/tests/testthat/test_bundle.R b/tests/testthat/test_bundle.R new file mode 100644 index 000000000..7f722fb92 --- /dev/null +++ b/tests/testthat/test_bundle.R @@ -0,0 +1,83 @@ +test_that("bundleable learner behaves as expected", { + task = tsk("mtcars") + LearnerRegrTest = R6Class("LearnerRegrTest", + inherit = LearnerRegrFeatureless, + public = list( + initialize = function() { + super$initialize() + self$id = "regr.test" + self$properties = unique(self$properties, "bundle") + }, + bundle = function() { + learner_bundle(self) + }, + unbundle = function() { + learner_unbundle(self) + } + ), + private = list( + .bundle = function(model) { + private$.tmp_model = model + "bundle" + }, + .unbundle = function(model) { + model = private$.tmp_model + private$.tmp_model = NULL + private$.counter = private$.counter + 1 + model + }, + .tmp_model = NULL, + .counter = 0 + ), + active = list( + bundled = function() { + learner_bundled(self) + } + ) + ) + learner = LearnerRegrTest$new() + # bundleable learner + expect_true("bundle" %in% learner$properties) + expect_bundleable(learner, task) + + # callr encapsulation causes bundling + learner2 = LearnerRegrTest$new() + learner2$encapsulate = c(train = "callr") + learner2$train(task) + + learner3 = LearnerRegrTest$new() + learner3$encapsulate = c(train = "try") + learner3$train(task) + expect_false(learner3$bundled) + + # for callr, we had to unbundle + expect_equal(get_private(learner2)$.counter, 1) + # for other encapsulation, no need to unbundle becausse it was not bundled + expect_equal(get_private(learner3)$.counter, 0) + + learner4 = LearnerRegrTest$new() + learner4$train(task) + # invisible + expect_invisible(learner$bundle()) + expect_invisible(learner$unbundle()) + + # bundling ResampleResult without stored models does not break state + rr = resample(task, learner4, rsmp("cv"), store_models = FALSE) + s1 = map(rr$learners, function(x) x$state) + rr$bundle() + s2 = map(rr$learners, function(x) x$state) + expect_equal(s1, s2) + + # bundles is FALSE when model was nulled + learner5 = LearnerRegrTest$new() + learner5$train(task) + learner5$bundle() + learner5$model = NULL + expect_false(learner5$bundled) +}) + +test_that("unbundleable learner's bundle is NULL", { + learner = lrn("regr.featureless") + learner$train(tsk("mtcars")) + expect_true(is.null(learner$state$bundled)) +}) diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index 48c38f87f..cf14b1753 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -161,15 +161,19 @@ test_that("bundling", { task = tsk("mtcars") LearnerRegrTest = R6Class("LearnerRegrTest", inherit = LearnerRegrFeatureless, + public = list( + bundle = function() learner_bundle(self), + unbundle = function() learner_unbundle(self) + ), + active = list( + bundled = function() learner_bundled(self) + ), private = list( .bundle = function(model) { - private$.tmp_model = model - "bundle" + structure(list(model), class = "bundled") }, .unbundle = function(model) { - model = private$.tmp_model - private$.tmp_model = NULL - model + model[[1L]] }, .tmp_model = NULL ) @@ -177,12 +181,31 @@ test_that("bundling", { learner = LearnerRegrTest$new() # allow to bundle during resample() - resampling = rsmp("holdout")$instantiate(task) + resampling = rsmp("cv", folds = 2L) + resampling$instantiate(task) # to compare different resample results rr1 = resample(task, learner, resampling, store_models = TRUE, bundle = TRUE) lrn_rec = rr1$learners[[1L]] expect_true(lrn_rec$bundled) expect_false(lrn_rec$unbundle()$bundled) + # ResampleResult can be unbundled. + expect_true(rr1$learners[[1]]$bundled) + expect_class(rr1$learners[[1]]$model, "bundled") + expect_true(rr1$learners[[2]]$bundled) + expect_class(rr1$learners[[2]]$model, "bundled") + + rr1$unbundle() + expect_false(rr1$learners[[1]]$bundled) + expect_class(rr1$learners[[1]]$model, "regr.featureless_model") + expect_false(rr1$learners[[2]]$bundled) + expect_class(rr1$learners[[2]]$model, "regr.featureless_model") + + rr1$bundle() + expect_true(rr1$learners[[1]]$bundled) + expect_class(rr1$learners[[1]]$model, "bundled") + expect_true(rr1$learners[[2]]$bundled) + expect_class(rr1$learners[[2]]$model, "bundled") + # bundled resamples are equivalent to unbundled results rr2 = resample(task, lrn("regr.featureless"), resampling, store_models = TRUE) rr3 = resample(task, learner, resampling, store_models = FALSE) @@ -193,3 +216,11 @@ test_that("bundling", { rr4 = resample(task, learner, resampling, bundle = FALSE, store_models = TRUE) expect_false(rr4$learners[[1]]$bundled) }) + +test_that("bundling does nothing with unbundleable learners", { + rr = resample(tsk("mtcars"), lrn("regr.featureless"), rsmp("cv", folds = 2)) + rr$unbundle() + expect_resample_result(rr) + rr$bundle() + expect_resample_result(rr) +}) From ea651e229ea8b87ec885500f1f70f6857c741b70 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 31 Jan 2024 19:26:46 +0100 Subject: [PATCH 06/47] bundle property must be manually set --- R/Learner.R | 4 ---- 1 file changed, 4 deletions(-) diff --git a/R/Learner.R b/R/Learner.R index a1ae3941e..576fed57d 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -164,10 +164,6 @@ Learner = R6Class("Learner", self$predict_types = assert_ordered_set(predict_types, names(mlr_reflections$learner_predict_types[[task_type]]), empty.ok = FALSE, .var.name = "predict_types") private$.predict_type = predict_types[1L] - - if (!is.null(private$.bundle) && !is.null(private$.unbundle)) { - properties = unique(c("bundle", properties)) - } self$properties = sort(assert_subset(properties, mlr_reflections$learner_properties[[task_type]])) self$data_formats = assert_subset(data_formats, mlr_reflections$data_formats) self$packages = union("mlr3", assert_character(packages, any.missing = FALSE, min.chars = 1L)) From 7d767de69495e70037eb15b23e97fa03f908b623 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 31 Jan 2024 19:45:10 +0100 Subject: [PATCH 07/47] fix tests --- tests/testthat/test_benchmark.R | 1 + tests/testthat/test_bundle.R | 2 +- tests/testthat/test_resample.R | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index 7ea86d417..88636cf45 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -486,6 +486,7 @@ test_that("bundling", { initialize = function(class = "bundle") { private$.class = class super$initialize() + self$properties = c("bundle", self$properties) }, bundle = function() { learner_bundle(self) diff --git a/tests/testthat/test_bundle.R b/tests/testthat/test_bundle.R index 7f722fb92..5a5cc4abe 100644 --- a/tests/testthat/test_bundle.R +++ b/tests/testthat/test_bundle.R @@ -6,7 +6,7 @@ test_that("bundleable learner behaves as expected", { initialize = function() { super$initialize() self$id = "regr.test" - self$properties = unique(self$properties, "bundle") + self$properties = c("bundle", self$properties) }, bundle = function() { learner_bundle(self) diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index cf14b1753..c502db80d 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -162,6 +162,10 @@ test_that("bundling", { LearnerRegrTest = R6Class("LearnerRegrTest", inherit = LearnerRegrFeatureless, public = list( + initialize = function() { + super$initialize() + self$properties = c("bundle", self$properties) + }, bundle = function() learner_bundle(self), unbundle = function() learner_unbundle(self) ), From 291d450b27fd226b7ca2129744ef905c49f7e516 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 31 Jan 2024 19:47:26 +0100 Subject: [PATCH 08/47] better docs --- R/bundle.R | 6 ++++-- man/bundling.Rd | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/R/bundle.R b/R/bundle.R index 272b67bf4..b3b4202a5 100644 --- a/R/bundle.R +++ b/R/bundle.R @@ -8,11 +8,12 @@ #' #' The function: #' * `learner_bundle(learner)`: -#' Replaces the learner's model with the bundled model. +#' Replaces the learner's model with the bundled model (in-place). #' * `learner_unbundle(learner) : -#' Replaces the learner's model with the unbundled model. +#' Replaces the learner's model with the unbundled model (in-place). #' * `learner_bundled(learner)`: #' returns `FALSE` if the learner is either not trained or not bundled, otherwise `TRUE`. +#' Does not modify the learner. #' #' All three functions are primarily intended to be used when implementing bundling for a [`Learner`]. #' Users who want to (un)bundle a [`Learner`] can instead use the public methods `$bundle()` and `$unbundle()` or the @@ -27,6 +28,7 @@ #' without modifying the learner's state. Must not depend on the learner's state. #' * the private method `$.unbundle(model)`, which takes in a [`Learner`]'s bundled model and returns it in #' unbundled form. Must not depend on the learner's state. +#' * add the property `bundle` to the learner's properties. #' #' To test the bundling implementation, you can use the internal test helper `expect_bundleable()`. #' This is also run in `expect_learner()` when a task is provided. diff --git a/man/bundling.Rd b/man/bundling.Rd index dfc17b1a3..53564dbce 100644 --- a/man/bundling.Rd +++ b/man/bundling.Rd @@ -24,11 +24,12 @@ deserialized. The naming is inspired from \CRANpkg{bundle}. The function: \itemize{ \item \code{learner_bundle(learner)}: -Replaces the learner's model with the bundled model. +Replaces the learner's model with the bundled model (in-place). \item `learner_unbundle(learner) : -Replaces the learner's model with the unbundled model. +Replaces the learner's model with the unbundled model (in-place). \item \code{learner_bundled(learner)}: returns \code{FALSE} if the learner is either not trained or not bundled, otherwise \code{TRUE}. +Does not modify the learner. } All three functions are primarily intended to be used when implementing bundling for a \code{\link{Learner}}. From ff31f66f06d4fc7aec1128270a61a1146aa29eae Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 31 Jan 2024 19:52:55 +0100 Subject: [PATCH 09/47] fix one more test --- inst/testthat/helper_expectations.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index 352f22646..aaef795a6 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -399,7 +399,7 @@ expect_bundleable = function(learner, task) { exists(x, learner, inherits = FALSE) } has_private = function(learner, x) { - exists(x, get_private(learner), inherits = FALSE) + exists(x, mlr3misc::get_private(learner), inherits = FALSE) } expect_true(has_public(learner, "bundle") && test_function(learner$bundle, nargs = 0)) From 2152b754d3016f9ef7ab587aaf060717d968adf3 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 31 Jan 2024 20:02:52 +0100 Subject: [PATCH 10/47] really fix test --- inst/testthat/helper_expectations.R | 4 ++-- man/bundling.Rd | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index aaef795a6..9634f48ea 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -405,8 +405,8 @@ expect_bundleable = function(learner, task) { expect_true(has_public(learner, "bundle") && test_function(learner$bundle, nargs = 0)) expect_true(has_public(learner, "unbundle") && test_function(learner$unbundle, nargs = 0)) expect_true(has_public(learner, "bundle")) - expect_true(has_private(learner, ".bundle") && test_function(get_private(learner)$.bundle, nargs = 1, args = "model")) - expect_true(has_private(learner, ".unbundle") && test_function(get_private(learner)$.unbundle, nargs = 1, args = "model")) + expect_true(has_private(learner, ".bundle") && test_function(mlr3misc::get_private(learner)$.bundle, nargs = 1, args = "model")) + expect_true(has_private(learner, ".unbundle") && test_function(mlr3misc::get_private(learner)$.unbundle, nargs = 1, args = "model")) expect_false(learner$bundled) diff --git a/man/bundling.Rd b/man/bundling.Rd index 53564dbce..5c7413725 100644 --- a/man/bundling.Rd +++ b/man/bundling.Rd @@ -47,6 +47,7 @@ In order to implement bundling for a \code{\link{Learner}}, you need to add: without modifying the learner's state. Must not depend on the learner's state. \item the private method \verb{$.unbundle(model)}, which takes in a \code{\link{Learner}}'s bundled model and returns it in unbundled form. Must not depend on the learner's state. +\item add the property \code{bundle} to the learner's properties. } To test the bundling implementation, you can use the internal test helper \code{expect_bundleable()}. From 9bd2efe00f7585feada62f91ec67c376e8b00cdc Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 31 Jan 2024 20:29:42 +0100 Subject: [PATCH 11/47] public methods --- R/ResultData.R | 4 ++-- R/bundle.R | 10 +++++----- R/worker.R | 2 +- inst/testthat/helper_expectations.R | 7 ++----- man/bundling.Rd | 6 +++--- tests/testthat/test_benchmark.R | 12 ++++++------ tests/testthat/test_bundle.R | 12 ++++++------ tests/testthat/test_resample.R | 14 +++++++------- 8 files changed, 32 insertions(+), 35 deletions(-) diff --git a/R/ResultData.R b/R/ResultData.R index b54d8d83d..76608dc13 100644 --- a/R/ResultData.R +++ b/R/ResultData.R @@ -250,7 +250,7 @@ ResultData = R6Class("ResultData", phash = self$data$fact[i, "learner_phash"][[1L]] learner = self$data$learners[phash, "learner", on = "learner_phash"][[1L]][[1L]] if (!is.null(state$model) && isFALSE(state$bundled)) { - state$model = get_private(learner)$.bundle(state$model) + state$model = learner$bundle_model(state$model) state$bundled = TRUE } state @@ -266,7 +266,7 @@ ResultData = R6Class("ResultData", phash = self$data$fact[i, "learner_phash"][[1L]] learner = self$data$learners[phash, "learner", on = "learner_phash"][[1L]][[1L]] if (!is.null(state$model) && isTRUE(state$bundled)) { - state$model = get_private(learner)$.unbundle(state$model) + state$model = learner$unbundle_model(state$model) state$bundled = FALSE } state diff --git a/R/bundle.R b/R/bundle.R index b3b4202a5..c1ad135b6 100644 --- a/R/bundle.R +++ b/R/bundle.R @@ -23,11 +23,11 @@ #' In order to implement bundling for a [`Learner`], you need to add: #' * the public methods `$bundle()` and `$unbundle()`, where you call `learner_bundle(self)` and #' `learner_unbundle(self)` respectively. -#' * the active binding `$bundled`, where you simply call `learner_bundled(self)`. -#' * the private method `$.bundle(model)`, which takes in a [`Learner`]'s model and returns it in bundled form, +#' * the public method `$bundle_model(model)`, which takes in a [`Learner`]'s model and returns it in bundled form, #' without modifying the learner's state. Must not depend on the learner's state. -#' * the private method `$.unbundle(model)`, which takes in a [`Learner`]'s bundled model and returns it in +#' * the public method `$unbundle_model(model)`, which takes in a [`Learner`]'s bundled model and returns it in #' unbundled form. Must not depend on the learner's state. +#' * the active binding `$bundled`, where you simply call `learner_bundled(self)`. #' * add the property `bundle` to the learner's properties. #' #' To test the bundling implementation, you can use the internal test helper `expect_bundleable()`. @@ -47,7 +47,7 @@ learner_unbundle = function(learner) { if (isFALSE(learner$bundled)) { warningf("Learner '%s' has not been bundled, skipping.", learner$id) } else if (isTRUE(learner$bundled)) { - learner$model = get_private(learner)$.unbundle(learner$model) + learner$model = learner$unbundle_model(learner$model) learner$state$bundled = FALSE } invisible(learner) @@ -62,7 +62,7 @@ learner_bundle = function(learner) { if (isTRUE(learner$bundled)) { warningf("Learner '%s' has already been bundled, skipping.", learner$id) } else if ("bundle" %in% learner$properties) { - learner$model = get_private(learner)$.bundle(learner$model) + learner$model = learner$bundle_model(learner$model) learner$state$bundled = TRUE } invisible(learner) diff --git a/R/worker.R b/R/worker.R index 61065b2e9..099c3794f 100644 --- a/R/worker.R +++ b/R/worker.R @@ -19,7 +19,7 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL } if ("bundle" %in% learner$properties && identical(learner$encapsulate[["train"]], "callr")) { - model = get_private(learner)$.bundle(model) + model = learner$bundle_model(model) } model diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index 9634f48ea..1eb12a151 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -398,15 +398,12 @@ expect_bundleable = function(learner, task) { has_public = function(learner, x) { exists(x, learner, inherits = FALSE) } - has_private = function(learner, x) { - exists(x, mlr3misc::get_private(learner), inherits = FALSE) - } expect_true(has_public(learner, "bundle") && test_function(learner$bundle, nargs = 0)) expect_true(has_public(learner, "unbundle") && test_function(learner$unbundle, nargs = 0)) expect_true(has_public(learner, "bundle")) - expect_true(has_private(learner, ".bundle") && test_function(mlr3misc::get_private(learner)$.bundle, nargs = 1, args = "model")) - expect_true(has_private(learner, ".unbundle") && test_function(mlr3misc::get_private(learner)$.unbundle, nargs = 1, args = "model")) + expect_true(has_public(learner, "bundle_model") && test_function(learner$bundle_model, nargs = 1, args = "model")) + expect_true(has_public(learner, "unbundle_model") && test_function(learner$unbundle_model, nargs = 1, args = "model")) expect_false(learner$bundled) diff --git a/man/bundling.Rd b/man/bundling.Rd index 5c7413725..18b8c336c 100644 --- a/man/bundling.Rd +++ b/man/bundling.Rd @@ -42,11 +42,11 @@ In order to implement bundling for a \code{\link{Learner}}, you need to add: \itemize{ \item the public methods \verb{$bundle()} and \verb{$unbundle()}, where you call \code{learner_bundle(self)} and \code{learner_unbundle(self)} respectively. -\item the active binding \verb{$bundled}, where you simply call \code{learner_bundled(self)}. -\item the private method \verb{$.bundle(model)}, which takes in a \code{\link{Learner}}'s model and returns it in bundled form, +\item the public method \verb{$bundle_model(model)}, which takes in a \code{\link{Learner}}'s model and returns it in bundled form, without modifying the learner's state. Must not depend on the learner's state. -\item the private method \verb{$.unbundle(model)}, which takes in a \code{\link{Learner}}'s bundled model and returns it in +\item the public method \verb{$unbundle_model(model)}, which takes in a \code{\link{Learner}}'s bundled model and returns it in unbundled form. Must not depend on the learner's state. +\item the active binding \verb{$bundled}, where you simply call \code{learner_bundled(self)}. \item add the property \code{bundle} to the learner's properties. } diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index 88636cf45..72cff51c3 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -493,6 +493,12 @@ test_that("bundling", { }, unbundle = function() { learner_unbundle(self) + }, + bundle_model = function(model) { + structure(list(model), class = private$.class) + }, + unbundle_model = function(model) { + model[[1L]] } ), active = list( @@ -501,12 +507,6 @@ test_that("bundling", { } ), private = list( - .bundle = function(model) { - structure(list(model), class = private$.class) - }, - .unbundle = function(model) { - model[[1L]] - }, .class = NULL ) ) diff --git a/tests/testthat/test_bundle.R b/tests/testthat/test_bundle.R index 5a5cc4abe..11aa72a8c 100644 --- a/tests/testthat/test_bundle.R +++ b/tests/testthat/test_bundle.R @@ -13,19 +13,19 @@ test_that("bundleable learner behaves as expected", { }, unbundle = function() { learner_unbundle(self) - } - ), - private = list( - .bundle = function(model) { + }, + bundle_model = function(model) { private$.tmp_model = model "bundle" }, - .unbundle = function(model) { + unbundle_model = function(model) { model = private$.tmp_model private$.tmp_model = NULL private$.counter = private$.counter + 1 model - }, + } + ), + private = list( .tmp_model = NULL, .counter = 0 ), diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index c502db80d..d0b4cc1ee 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -167,18 +167,18 @@ test_that("bundling", { self$properties = c("bundle", self$properties) }, bundle = function() learner_bundle(self), - unbundle = function() learner_unbundle(self) + unbundle = function() learner_unbundle(self), + bundle_model = function(model) { + structure(list(model), class = "bundled") + }, + unbundle_model = function(model) { + model[[1L]] + } ), active = list( bundled = function() learner_bundled(self) ), private = list( - .bundle = function(model) { - structure(list(model), class = "bundled") - }, - .unbundle = function(model) { - model[[1L]] - }, .tmp_model = NULL ) ) From cfbab13582527dedcfb03a0fb15099d0cb656390 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Thu, 1 Feb 2024 19:10:24 +0100 Subject: [PATCH 12/47] refactor --- DESCRIPTION | 3 +- NAMESPACE | 14 ++- NEWS.md | 4 +- R/BenchmarkResult.R | 12 +- R/HotstartStack.R | 11 +- R/Learner.R | 6 +- R/LearnerClassifLily.R | 74 ++++++++++++ R/Measure.R | 5 + R/ResampleResult.R | 12 +- R/ResultData.R | 34 ++---- R/benchmark.R | 15 ++- R/bundle.R | 75 ------------ R/helper_exec.R | 3 +- R/marshal.R | 105 +++++++++++++++++ R/mlr_reflections.R | 2 +- R/resample.R | 15 ++- R/worker.R | 31 +++-- inst/testthat/helper_expectations.R | 63 +++++----- man-roxygen/param_bundle.R | 3 - man-roxygen/param_learner_properties.R | 4 +- man-roxygen/param_unmarshal.R | 2 + man/BenchmarkResult.Rd | 24 ++-- man/Learner.Rd | 4 +- man/LearnerClassif.Rd | 4 +- man/LearnerRegr.Rd | 4 +- man/ResampleResult.Rd | 24 ++-- man/ResultData.Rd | 24 ++-- man/benchmark.Rd | 7 +- man/bundling.Rd | 60 ---------- man/marshalling.Rd | 64 ++++++++++ man/mlr_learners_classif.lily.Rd | 135 ++++++++++++++++++++++ man/resample.Rd | 7 +- tests/testthat/test_HotstartStack.R | 8 ++ tests/testthat/test_Learner.R | 17 +++ tests/testthat/test_LearnerClassifLily.R | 34 ++++++ tests/testthat/test_Measure.R | 10 ++ tests/testthat/test_benchmark.R | 141 ++++++++++------------- tests/testthat/test_bundle.R | 83 ------------- tests/testthat/test_marshal.R | 18 +++ tests/testthat/test_resample.R | 128 ++++++++++---------- 40 files changed, 758 insertions(+), 531 deletions(-) create mode 100644 R/LearnerClassifLily.R delete mode 100644 R/bundle.R create mode 100644 R/marshal.R delete mode 100644 man-roxygen/param_bundle.R create mode 100644 man-roxygen/param_unmarshal.R delete mode 100644 man/bundling.Rd create mode 100644 man/marshalling.Rd create mode 100644 man/mlr_learners_classif.lily.Rd create mode 100644 tests/testthat/test_LearnerClassifLily.R delete mode 100644 tests/testthat/test_bundle.R create mode 100644 tests/testthat/test_marshal.R diff --git a/DESCRIPTION b/DESCRIPTION index 9793e8490..23e4c77ca 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -90,6 +90,7 @@ Collate: 'mlr_learners.R' 'LearnerClassifDebug.R' 'LearnerClassifFeatureless.R' + 'LearnerClassifLily.R' 'LearnerClassifRpart.R' 'LearnerRegr.R' 'LearnerRegrDebug.R' @@ -175,7 +176,6 @@ Collate: 'benchmark.R' 'benchmark_grid.R' 'bibentries.R' - 'bundle.R' 'default_measures.R' 'fix_factor_levels.R' 'helper.R' @@ -184,6 +184,7 @@ Collate: 'helper_hashes.R' 'helper_print.R' 'install_pkgs.R' + 'marshal.R' 'mlr_sugar.R' 'mlr_test_helpers.R' 'partition.R' diff --git a/NAMESPACE b/NAMESPACE index e52c2bddf..43c14bfca 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -87,6 +87,8 @@ S3method(fix_factor_levels,data.table) S3method(head,Task) S3method(is_missing_prediction_data,PredictionDataClassif) S3method(is_missing_prediction_data,PredictionDataRegr) +S3method(marshal_model,classif_lily_model) +S3method(marshal_model,default) S3method(partition,Task) S3method(partition,TaskClassif) S3method(partition,TaskRegr) @@ -104,6 +106,8 @@ S3method(set_threads,default) S3method(set_threads,list) S3method(summary,Task) S3method(tail,Task) +S3method(unmarshal_model,classif_lily_model_marshalled) +S3method(unmarshal_model,default) export(BenchmarkResult) export(DataBackend) export(DataBackendDataTable) @@ -113,6 +117,7 @@ export(Learner) export(LearnerClassif) export(LearnerClassifDebug) export(LearnerClassifFeatureless) +export(LearnerClassifLily) export(LearnerClassifRpart) export(LearnerRegr) export(LearnerRegrDebug) @@ -208,11 +213,13 @@ export(extract_pkgs) export(filter_prediction_data) export(install_pkgs) export(is_missing_prediction_data) -export(learner_bundle) -export(learner_bundled) -export(learner_unbundle) +export(learner_marshal) +export(learner_marshalled) +export(learner_unmarshal) export(lrn) export(lrns) +export(marshal_model) +export(marshalled_model) export(mlr_learners) export(mlr_measures) export(mlr_reflections) @@ -230,6 +237,7 @@ export(tgen) export(tgens) export(tsk) export(tsks) +export(unmarshal_model) import(checkmate) import(data.table) import(mlr3misc) diff --git a/NEWS.md b/NEWS.md index e6e2cdcf7..c86d3d674 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,8 @@ # mlr3 (development version) -* Feat: added support for `"bundle"` property, which allows learners to process +* Feat: added support for `"marshal"` property, which allows learners to process models so they can be serialized. This happens automatically during `resample()` -and `benchmark()`. The naming was inspired by the {bundle} package. +and `benchmark()`. The naming was inspired by the {marshal} package. # mlr3 0.17.2 diff --git a/R/BenchmarkResult.R b/R/BenchmarkResult.R index caf8cfec5..f33290f9b 100644 --- a/R/BenchmarkResult.R +++ b/R/BenchmarkResult.R @@ -134,14 +134,14 @@ BenchmarkResult = R6Class("BenchmarkResult", }, #' @description - #' Bundles all stored models. - bundle = function() { - private$.data$bundle() + #' marshals all stored models. + marshal = function() { + private$.data$marshal() }, #' @description - #' Unbundles all stored models. - unbundle = function() { - private$.data$unbundle() + #' Unmarshals all stored models. + unmarshal = function() { + private$.data$unmarshal() }, #' @description diff --git a/R/HotstartStack.R b/R/HotstartStack.R index 11b845915..431b28c9e 100644 --- a/R/HotstartStack.R +++ b/R/HotstartStack.R @@ -85,10 +85,13 @@ HotstartStack = R6Class("HotstartStack", add = function(learners) { learners = assert_learners(as_learners(learners)) - # check for models - if (any(map_lgl(learners, function(learner) is.null(learner$state$model)))) { - stopf("Learners must be trained before adding them to the hotstart stack.") - } + walk(learners, function(learner) { + if (is.null(learner$model)) { + stopf("Learners must be trained before adding them to the hotstart stack.") + } else if (marshalled_model(learner$model)) { + stopf("Learners must be unmarshalled before adding them to the hotstart stack.") + } + }) if (!is.null(self$hotstart_threshold)) { learners = keep(learners, function(learner) { diff --git a/R/Learner.R b/R/Learner.R index 576fed57d..7ef30bae8 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -184,7 +184,7 @@ Learner = R6Class("Learner", #' @param ... (ignored). print = function(...) { catn(format(self), if (is.null(self$label) || is.na(self$label)) "" else paste0(": ", self$label)) - catn(str_indent("* Model:", if (is.null(self$model)) "-" else if (learner_bundled(self)) "" else paste0(class(self$model)[1L]))) + catn(str_indent("* Model:", if (is.null(self$model)) "-" else if (marshalled_model(self$model)) "" else paste0(class(self$model)[1L]))) catn(str_indent("* Parameters:", as_short_string(self$param_set$values, 1000L))) catn(str_indent("* Packages:", self$packages)) catn(str_indent("* Predict Types: ", replace(self$predict_types, self$predict_types == self$predict_type, paste0("[", self$predict_type, "]")))) @@ -279,8 +279,8 @@ Learner = R6Class("Learner", stopf("Cannot predict, Learner '%s' has not been trained yet", self$id) } - if (isTRUE(self$state$bundled)) { - stopf("Cannot predict, Learner '%s' has not been unbundled yet", self$id) + if (marshalled_model(self$model)) { + stopf("Cannot predict, Learner '%s' has not been unmarshalled yet", self$id) } if (isTRUE(self$parallel_predict) && nbrOfWorkers() > 1L) { diff --git a/R/LearnerClassifLily.R b/R/LearnerClassifLily.R new file mode 100644 index 000000000..36440a39d --- /dev/null +++ b/R/LearnerClassifLily.R @@ -0,0 +1,74 @@ +#' @title Lily and Marshall +#' +#' @name mlr_learners_classif.lily +#' @include LearnerClassifDebug.R +#' +#' @description +#' This learner is just like [`LearnerClassifDebug`], but can be marshalled. +#' When the `count_marshalling` parameter is `TRUE`, the model contains a `marshal_count` that will be increased +#' by 1, each time `marshal_model` is called. +#' +#' @templateVar id classif.lily +#' @template learner +#' +#' @export +LearnerClassifLily = R6Class("LearnerClassifLily", + inherit = LearnerClassifDebug, + public = list( + #' @description + #' Creates a new instance of this [R6][R6::R6Class] class. + initialize = function() { + super$initialize() + self$param_set$add(ps(count_marshalling = p_lgl(tags = c("train", "required")))) + self$param_set$values$count_marshalling = FALSE + self$properties = sort(c("marshal", self$properties)) + self$man = "mlr3::mlr_learners_classif.lily" + self$label = "Lily Learner" + self$id = "classif.lily" + }, + #' @description + #' Marshals the learner. + marshal = function() { + learner_marshal(self) + }, + #' @description + #' Unmarshal the learner. + unmarshal = function() { + learner_unmarshal(self) + } + ), + active = list( + #' @field marshalled (logical(1))\cr + #' Whether the learner has been marshalled. + marshalled = function() { + learner_marshalled(self) + } + ), + private = list( + .train = function(task) { + model = super$.train(task) + if (self$param_set$values$count_marshalling) { + model$marshal_count = 0L + } + class(model) = "classif_lily_model" + return(model) + } + ) +) + +#' @include mlr_learners.R +mlr_learners$add("classif.lily", function() LearnerClassifLily$new()) + +#' @export +marshal_model.classif_lily_model = function(model, ...) { + if (!is.null(model$marshal_count)) { + model$marshal_count = model$marshal_count + 1 + } + newclass = c("classif_lily_model_marshalled", "marshalled") + structure(list(model), class = newclass) +} + +#' @export +unmarshal_model.classif_lily_model_marshalled = function(model, ...) { + model[[1L]] +} diff --git a/R/Measure.R b/R/Measure.R index d781235c0..2169bf8ce 100644 --- a/R/Measure.R +++ b/R/Measure.R @@ -173,6 +173,8 @@ Measure = R6Class("Measure", assert_measure(self, task = task, learner = learner) assert_prediction(prediction) + # FIXME: if self has property model check that not marshalled + if ("requires_task" %in% self$properties && is.null(task)) { stopf("Measure '%s' requires a task", self$id) } @@ -184,6 +186,9 @@ Measure = R6Class("Measure", if ("requires_model" %in% self$properties && (is.null(learner) || is.null(learner$model))) { stopf("Measure '%s' requires the trained model", self$id) } + if ("requires_model" %in% self$properties && marshalled_model(learner$model)) { + stopf("Measure '%s' requires the trained model, but model is un marshalled form", self$id) + } if ("requires_train_set" %in% self$properties && is.null(train_set)) { stopf("Measure '%s' requires the train_set", self$id) diff --git a/R/ResampleResult.R b/R/ResampleResult.R index 7f032c22e..fc15d4d4c 100644 --- a/R/ResampleResult.R +++ b/R/ResampleResult.R @@ -225,14 +225,14 @@ ResampleResult = R6Class("ResampleResult", }, #' @description - #' Bundles all stored learner models. - bundle = function() { - private$.data$bundle() + #' marshals all stored learner models. + marshal = function() { + private$.data$marshal() }, #' @description - #' Unbundles all stored learner models. - unbundle = function() { - private$.data$unbundle() + #' Unmarshals all stored learner models. + unmarshal = function() { + private$.data$unmarshal() } ), diff --git a/R/ResultData.R b/R/ResultData.R index 76608dc13..f3d7c6072 100644 --- a/R/ResultData.R +++ b/R/ResultData.R @@ -243,35 +243,17 @@ ResultData = R6Class("ResultData", }, #' @description - #' Bundles all stored learner models. - bundle = function() { - learner_states = map(seq_len(nrow(self$data$fact)), function(i) { - state = self$data$fact[i, "learner_state"][[1L]][[1L]] - phash = self$data$fact[i, "learner_phash"][[1L]] - learner = self$data$learners[phash, "learner", on = "learner_phash"][[1L]][[1L]] - if (!is.null(state$model) && isFALSE(state$bundled)) { - state$model = learner$bundle_model(state$model) - state$bundled = TRUE - } - state - }) - self$data$fact$learner_state = learner_states + #' Marshals all stored learner models. + marshal = function() { + learner_state = NULL + self$data$fact[, learner_state := lapply(learner_state, marshal_state)] invisible(self) }, #' @description - #' Unbundles all stored learner models. - unbundle = function() { - learner_states = map(seq_len(nrow(self$data$fact)), function(i) { - state = self$data$fact[i, "learner_state"][[1L]][[1L]] - phash = self$data$fact[i, "learner_phash"][[1L]] - learner = self$data$learners[phash, "learner", on = "learner_phash"][[1L]][[1L]] - if (!is.null(state$model) && isTRUE(state$bundled)) { - state$model = learner$unbundle_model(state$model) - state$bundled = FALSE - } - state - }) - self$data$fact$learner_state = learner_states + #' Unmarshals all stored learner models. + unmarshal = function() { + learner_state = NULL + self$data$fact[, learner_state := lapply(learner_state, unmarshal_state)] invisible(self) }, diff --git a/R/benchmark.R b/R/benchmark.R index cfe62d127..462aeb64e 100644 --- a/R/benchmark.R +++ b/R/benchmark.R @@ -14,7 +14,7 @@ #' @template param_encapsulate #' @template param_allow_hotstart #' @template param_clone -#' @template param_bundle +#' @template param_unmarshal #' #' @return [BenchmarkResult]. #' @@ -78,7 +78,7 @@ #' ## Get the training set of the 2nd iteration of the featureless learner on penguins #' rr = bmr$aggregate()[learner_id == "classif.featureless"]$resample_result[[1]] #' rr$resampling$train_set(2) -benchmark = function(design, store_models = FALSE, store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling"), bundle = TRUE) { +benchmark = function(design, store_models = FALSE, store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling"), unmarshal = TRUE) { assert_subset(clone, c("task", "learner", "resampling")) assert_data_frame(design, min.rows = 1L) assert_names(names(design), must.include = c("task", "learner", "resampling")) @@ -184,7 +184,7 @@ benchmark = function(design, store_models = FALSE, store_backends = TRUE, encaps res = future_map(n, workhorse, task = grid$task, learner = grid$learner, resampling = grid$resampling, iteration = grid$iteration, param_values = grid$param_values, mode = grid$mode, - MoreArgs = list(store_models = store_models, lgr_threshold = lgr_threshold, pb = pb, bundle = bundle) + MoreArgs = list(store_models = store_models, lgr_threshold = lgr_threshold, pb = pb) ) grid = insert_named(grid, list( @@ -197,5 +197,12 @@ benchmark = function(design, store_models = FALSE, store_backends = TRUE, encaps lg$info("Finished benchmark") set(grid, j = "mode", value = NULL) - BenchmarkResult$new(ResultData$new(grid, store_backends = store_backends)) + + result_data = ResultData$new(grid, store_backends = store_backends) + + if (unmarshal && store_models) { + result_data$unmarshal() + } + + BenchmarkResult$new(result_data) } diff --git a/R/bundle.R b/R/bundle.R deleted file mode 100644 index c1ad135b6..000000000 --- a/R/bundle.R +++ /dev/null @@ -1,75 +0,0 @@ -#' @title (Un)bundle a Learner -#' -#' @name bundling -#' -#' @description -#' Bundling is the process of processing the model of a trained [`Learner`] so it an be successfully serialized and -#' deserialized. The naming is inspired from \CRANpkg{bundle}. -#' -#' The function: -#' * `learner_bundle(learner)`: -#' Replaces the learner's model with the bundled model (in-place). -#' * `learner_unbundle(learner) : -#' Replaces the learner's model with the unbundled model (in-place). -#' * `learner_bundled(learner)`: -#' returns `FALSE` if the learner is either not trained or not bundled, otherwise `TRUE`. -#' Does not modify the learner. -#' -#' All three functions are primarily intended to be used when implementing bundling for a [`Learner`]. -#' Users who want to (un)bundle a [`Learner`] can instead use the public methods `$bundle()` and `$unbundle()` or the -#' field `$bundled`, which is more in line with `mlr3`'s object oriented design. -#' -#' @section Implementing Bundling for a Learner: -#' In order to implement bundling for a [`Learner`], you need to add: -#' * the public methods `$bundle()` and `$unbundle()`, where you call `learner_bundle(self)` and -#' `learner_unbundle(self)` respectively. -#' * the public method `$bundle_model(model)`, which takes in a [`Learner`]'s model and returns it in bundled form, -#' without modifying the learner's state. Must not depend on the learner's state. -#' * the public method `$unbundle_model(model)`, which takes in a [`Learner`]'s bundled model and returns it in -#' unbundled form. Must not depend on the learner's state. -#' * the active binding `$bundled`, where you simply call `learner_bundled(self)`. -#' * add the property `bundle` to the learner's properties. -#' -#' To test the bundling implementation, you can use the internal test helper `expect_bundleable()`. -#' This is also run in `expect_learner()` when a task is provided. -#' -#' For a concrete example on how to implement bundling, see the `LearnerTorch` class from -#' [mlr3torch](https://github.com/mlr-org/mlr3torch). -#' -#' @param learner [`Learner`]\cr -#' The learner to bundle. -#' @keywords internal -#' @export -learner_unbundle = function(learner) { - if (is.null(learner$model)) { - stopf("Cannot unbundle, Learner '%s' has not been trained yet", learner$id) - } - if (isFALSE(learner$bundled)) { - warningf("Learner '%s' has not been bundled, skipping.", learner$id) - } else if (isTRUE(learner$bundled)) { - learner$model = learner$unbundle_model(learner$model) - learner$state$bundled = FALSE - } - invisible(learner) -} - -#' @rdname bundling -#' @export -learner_bundle = function(learner) { - if (is.null(learner$model)) { - stopf("Cannot bundle, Learner '%s' has not been trained yet", learner$id) - } - if (isTRUE(learner$bundled)) { - warningf("Learner '%s' has already been bundled, skipping.", learner$id) - } else if ("bundle" %in% learner$properties) { - learner$model = learner$bundle_model(learner$model) - learner$state$bundled = TRUE - } - invisible(learner) -} - -#' @rdname bundling -#' @export -learner_bundled = function(learner) { - isTRUE(learner$state$bundled) && !is.null(learner$model) -} diff --git a/R/helper_exec.R b/R/helper_exec.R index bf68f13a9..aa85c43a2 100644 --- a/R/helper_exec.R +++ b/R/helper_exec.R @@ -42,6 +42,7 @@ future_map = function(n, FUN, ..., MoreArgs = list()) { future.apply::future_mapply( FUN, ..., MoreArgs = MoreArgs, SIMPLIFY = FALSE, USE.NAMES = FALSE, future.globals = FALSE, future.packages = "mlr3", future.seed = TRUE, - future.scheduling = scheduling, future.chunk.size = chunk_size, future.stdout = stdout) + future.scheduling = scheduling, future.chunk.size = chunk_size, future.stdout = stdout + ) } } diff --git a/R/marshal.R b/R/marshal.R new file mode 100644 index 000000000..38625dc7e --- /dev/null +++ b/R/marshal.R @@ -0,0 +1,105 @@ +#' @title (Un)marshal a Learner +#' +#' @name marshalling +#' +#' @description +#' Marshalling is the process of processing the model of a trained [`Learner`] so it an be successfully serialized and +#' deserialized. The naming is inspired from package [marshall](https://github.com/HenrikBengtsson/marshal) and we +#' plan to fully migrate to this package once it is on CRAN. +#' The supported implementation until then should therfore be considered as a temporary solution and is likely +#' to change in the future. +#' +#' The central functions (and the only methods that are used by `mlr3` internally) are: +#' * the S3 generic `marshal_model(model, ...)`. +#' Which takes in a model and returns it in marshalled form. +#' The suffix `"_marshalled"` should be added to the class of the returned object and the root class must +#' be set to `"marshalled"`. +#' * the S3 generic `unmarshal_model(model, ...)`. +#' Which takes in a model and returns it in unmarshalled form. +#' The returned object must not inherit from class `"marshalled"`. +#' * the function `marshalled_model(model)`, which returns `TRUE` if the model inherits from class `"marshalled"` +#' and `FALSE` otherwise. +#' +#' In order to implement marshalling for a Learner, you only need to overload the `marshal_model` and `unmarshal_model` +#' methods and tag the learner with the `"marshal"` property accordingly. +#' +#' To make marshalling accessible in an R6-manner, you should also add the public methods `$marshal()`, `$unmarshal()` +#' and the active binding `$marshalled`. +#' To make this as convenient as possible, the functions `learner_marshal(learner)`, `learner_unmarshal(learner)` +#' and `learner_marshalled(learner)` are provided and can be called from within the public methods. +#' All three functions throw an error if the learner is not trained and otherwise call +#' `marshal_model()`, `unmarshal_model()` or `marshalled_model()` on the learner's model. +#' +#' You can verify whether you have correctly implemented marshalling by using the internal test helper +#' `expect_marshallable_learner()`. This is also run by `expect_learner()` if a task is provided. +#' +#' For a concrete example on how to implement marshalling, see [`LearnerClassifLily`]. +#' +#' @param learner [`Learner`]\cr +#' The learner to marshal. +#' @keywords internal +#' @export +learner_unmarshal = function(learner) { + if (is.null(learner$model)) { + stopf("Cannot unmarshal, Learner '%s' has not been trained yet", learner$id) + } + learner$model = unmarshal_model(learner$model) + invisible(learner) +} + +#' @rdname marshalling +#' @export +learner_marshal = function(learner) { + if (is.null(learner$model)) { + stopf("Cannot marshal, Learner '%s' has not been trained yet", learner$id) + } + learner$model = marshal_model(learner$model) + invisible(learner) +} + +#' @rdname marshalling +#' @export +learner_marshalled = function(learner) { + if (is.null(learner$model)) { + stopf("Cannot check marshalled status, Learner '%s' has not been trained yet", learner$id) + } + marshalled_model(learner$model) +} + +#' @rdname marshalling +#' @export +marshal_model = function(model, ...) { + UseMethod("marshal_model") +} + +#' @rdname marshalling +#' @export +unmarshal_model = function(model, ...) { + UseMethod("unmarshal_model") +} + +#' @rdname marshalling +#' @export +marshalled_model = function(model) { + test_class(model, "marshalled") +} + +#' @export +marshal_model.default = function(model, ...) { + model +} + +#' @export +unmarshal_model.default = function(model, ...) { + model +} + +marshal_state = function(state, ...) { + state$model = marshal_model(state$model) + state +} + +unmarshal_state = function(state, ...) { + state$model = unmarshal_model(state$model) + state +} diff --git a/R/mlr_reflections.R b/R/mlr_reflections.R index ebe30d99f..02d206401 100644 --- a/R/mlr_reflections.R +++ b/R/mlr_reflections.R @@ -125,7 +125,7 @@ local({ ) ### Learner - tmp = c("featureless", "missings", "weights", "importance", "selected_features", "oob_error", "loglik", "hotstart_forward", "hotstart_backward", "bundle") + tmp = c("featureless", "missings", "weights", "importance", "selected_features", "oob_error", "loglik", "hotstart_forward", "hotstart_backward", "marshal") mlr_reflections$learner_properties = list( classif = c(tmp, "twoclass", "multiclass"), regr = tmp diff --git a/R/resample.R b/R/resample.R index 5a3efdd79..df1f62057 100644 --- a/R/resample.R +++ b/R/resample.R @@ -14,7 +14,7 @@ #' @template param_encapsulate #' @template param_allow_hotstart #' @template param_clone -#' @template param_bundle +#' @template param_unmarshal #' @return [ResampleResult]. #' #' @template section_predict_sets @@ -55,7 +55,7 @@ #' bmr1 = as_benchmark_result(rr) #' bmr2 = as_benchmark_result(rr_featureless) #' print(bmr1$combine(bmr2)) -resample = function(task, learner, resampling, store_models = FALSE, store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling"), bundle = TRUE) { +resample = function(task, learner, resampling, store_models = FALSE, store_backends = TRUE, encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling"), unmarshal = TRUE) { assert_subset(clone, c("task", "learner", "resampling")) task = assert_task(as_task(task, clone = "task" %in% clone)) learner = assert_learner(as_learner(learner, clone = "learner" %in% clone)) @@ -63,6 +63,7 @@ resample = function(task, learner, resampling, store_models = FALSE, store_backe assert_flag(store_models) assert_flag(store_backends) assert_learnable(task, learner) + assert_flag(unmarshal) set_encapsulation(list(learner), encapsulate) if (!resampling$is_instantiated) { @@ -112,7 +113,7 @@ resample = function(task, learner, resampling, store_models = FALSE, store_backe } res = future_map(n, workhorse, iteration = seq_len(n), learner = grid$learner, mode = grid$mode, - MoreArgs = list(task = task, resampling = resampling, store_models = store_models, lgr_threshold = lgr_threshold, pb = pb, bundle = bundle) + MoreArgs = list(task = task, resampling = resampling, store_models = store_models, lgr_threshold = lgr_threshold, pb = pb) ) data = data.table( @@ -127,5 +128,11 @@ resample = function(task, learner, resampling, store_models = FALSE, store_backe learner_hash = map_chr(res, "learner_hash") ) - ResampleResult$new(ResultData$new(data, store_backends = store_backends)) + result_data = ResultData$new(data, store_backends = store_backends) + + if (unmarshal && store_models) { + result_data$unmarshal() + } + + ResampleResult$new(result_data) } diff --git a/R/worker.R b/R/worker.R index 099c3794f..f1bf25ca0 100644 --- a/R/worker.R +++ b/R/worker.R @@ -1,5 +1,5 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NULL, mode = "train") { - # This wrapper calls learner$train, and additionally performs some basic + # This wrapper calls learner$.train, and additionally performs some basic # checks that the training was successful. # Exceptions here are possibly encapsulated, so that they get captured # and turned into log messages. @@ -18,8 +18,9 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL stopf("Learner '%s' on task '%s' returned NULL during internal %s()", learner$id, task$id, mode) } - if ("bundle" %in% learner$properties && identical(learner$encapsulate[["train"]], "callr")) { - model = learner$bundle_model(model) + if (learner$encapsulate[["train"]] == "callr") { + # the default method of marshal_model does nothing + model = marshal_model(model) } model @@ -72,16 +73,14 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL log = append_log(NULL, "train", result$log$class, result$log$msg) train_time = result$elapsed - # callr encapsualtion causes dangling pointers between train and predict - bundled = if ("bundle" %in% learner$properties) { - identical(learner$encapsulate[["train"]], "callr") - } - + # unmarshal_model does nothing if the model was not marshalled + # We always want to unmarshal the model, because either: + # a) the user called $train() manually or + # b) we are within worker and still have to make a prediction learner$state = insert_named(learner$state, list( - model = result$result, + model = unmarshal_model(result$result), log = log, train_time = train_time, - bundled = bundled, param_vals = learner$param_set$values, task_hash = task$hash, feature_names = task$feature_names, @@ -111,10 +110,6 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL fb$id, learner = fb$clone()) } - if (isTRUE(bundled)) { - learner$unbundle() - } - learner } @@ -231,7 +226,7 @@ learner_predict = function(learner, task, row_ids = NULL) { } -workhorse = function(iteration, task, learner, resampling, param_values = NULL, lgr_threshold, store_models = FALSE, pb = NULL, mode = "train", is_sequential = TRUE, bundle = TRUE) { +workhorse = function(iteration, task, learner, resampling, param_values = NULL, lgr_threshold, store_models = FALSE, pb = NULL, mode = "train", is_sequential = TRUE) { if (!is.null(pb)) { pb(sprintf("%s|%s|i:%i", task$id, learner$id, iteration)) } @@ -280,9 +275,9 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, if (!store_models) { lg$debug("Erasing stored model for learner '%s'", learner$id) learner$state$model = NULL - } else if (bundle && "bundle" %in% learner$properties) { - lg$debug("Bundling model for learner '%s'", learner$id) - learner$bundle() + } else if (!is_sequential) { + lg$debug("Marshalling model for learner '%s' because of non-sequential execution", learner$id) + learner$model = marshal_model(learner$model) } list(learner_state = learner$state, prediction = pdatas, param_values = learner$param_set$values, learner_hash = learner_hash) diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index 1eb12a151..a499faf36 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -380,9 +380,11 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { checkmate::expect_subset(lrn$properties, mlr3::mlr_reflections$learner_properties[[task$task_type]]) testthat::expect_identical(lrn$task_type, task$task_type) - if ("bundle" %in% lrn$properties) { - expect_bundleable(lrn, task) + if ("marshal" %in% lrn$properties) { + expect_marshallable_learner(lrn, task) } + } else if ("marshal" %in% lrn$properties) { + message("Cannot test 'marshal' property of the learner as no task is provided.") } if (!inherits(lrn, "GraphLearner") && !inherits(lrn, "AutoTuner")) { # still not in pipelines, breaking check in mlr3tuning @@ -391,48 +393,51 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { } -expect_bundleable = function(learner, task) { - expect_true("bundle" %in% learner$properties) +expect_marshallable_learner = function(learner, task) { + expect_true("marshal" %in% learner$properties) learner$state = NULL has_public = function(learner, x) { exists(x, learner, inherits = FALSE) } - expect_true(has_public(learner, "bundle") && test_function(learner$bundle, nargs = 0)) - expect_true(has_public(learner, "unbundle") && test_function(learner$unbundle, nargs = 0)) - expect_true(has_public(learner, "bundle")) - expect_true(has_public(learner, "bundle_model") && test_function(learner$bundle_model, nargs = 1, args = "model")) - expect_true(has_public(learner, "unbundle_model") && test_function(learner$unbundle_model, nargs = 1, args = "model")) + expect_true(has_public(learner, "marshal") && test_function(learner$marshal, nargs = 0)) + expect_true(has_public(learner, "unmarshal") && test_function(learner$unmarshal, nargs = 0)) + expect_true(has_public(learner, "marshalled")) - expect_false(learner$bundled) - - # (un)bundling only possible after training - expect_error(learner$bundle(), "has not been trained") - expect_error(learner$unbundle(), "has not been trained") + # (un)marshal only possible after training + expect_error(learner$marshal(), "has not been trained") + expect_error(learner$unmarshal(), "has not been trained") + expect_error(learner$marshalled, "has not been trained") learner$train(task) model = learner$model - expect_false(learner$bundled) - learner$bundle() - expect_true(learner$bundled) - - # cannot predict with bundled learner - expect_error(learner$predict(task), "has not been unbundled") - expect_true(identical(learner$model, "bundle")) - - # unbundling works - learner$unbundle() - # can predict after unbundling + class_prev = class(model) + expect_false(learner$marshalled) + expect_equal(marshalled_model(learner$model), learner$marshalled) + expect_invisible(learner$marshal()) + expect_true(learner$marshalled) + expect_equal(marshalled_model(learner$model), learner$marshalled) + + # cannot predict with marshalled learner + expect_error(learner$predict(task), "has not been unmarshalled") + + # unmarshalling works + expect_invisible(learner$unmarshal()) + # can predict after unmarshalling expect_prediction(learner$predict(task)) # model is reset expect_equal(learner$model, model) - # bundled is set accordingly - expect_false(learner$bundled) + # marshalled is set accordingly + expect_false(learner$marshalled) - # when re-training, bundled is reset + expect_equal(class(learner$model), class_prev) + + # when re-training, marshalled is reset learner$predict(task) - expect_false(learner$train(task)$bundled) + expect_false(learner$train(task)$marshalled) + + } diff --git a/man-roxygen/param_bundle.R b/man-roxygen/param_bundle.R deleted file mode 100644 index 819d8a763..000000000 --- a/man-roxygen/param_bundle.R +++ /dev/null @@ -1,3 +0,0 @@ -#' @param bundle (`logical(1)`)\cr -#' Whether to bundle the learner(s) after the train-predict loop. -#' Default is `TRUE`. diff --git a/man-roxygen/param_learner_properties.R b/man-roxygen/param_learner_properties.R index c97d97de9..e02bcf946 100644 --- a/man-roxygen/param_learner_properties.R +++ b/man-roxygen/param_learner_properties.R @@ -7,5 +7,5 @@ #' * `"importance"`: The learner supports extraction of importance scores, i.e. comes with an `$importance()` extractor function (see section on optional extractors in [Learner]). #' * `"selected_features"`: The learner supports extraction of the set of selected features, i.e. comes with a `$selected_features()` extractor function (see section on optional extractors in [Learner]). #' * `"oob_error"`: The learner supports extraction of estimated out of bag error, i.e. comes with a `oob_error()` extractor function (see section on optional extractors in [Learner]). -#' * `"bundle"`: To save learners with this property, you need to call `$bundle()` first. -#' If a learner is in a bundled state, you call first need to call `$unbundle()` to use it's model, e.g. for prediction. +#' * `"marshal"`: To save learners with this property, you need to call `$marshal()` first. +#' If a learner is in a marshalled state, you call first need to call `$unmarshal()` to use it's model, e.g. for prediction. diff --git a/man-roxygen/param_unmarshal.R b/man-roxygen/param_unmarshal.R new file mode 100644 index 000000000..c950d6184 --- /dev/null +++ b/man-roxygen/param_unmarshal.R @@ -0,0 +1,2 @@ +#' @param unmarshal [`Learner`]\cr +#' Whether to unmarshal the learner. diff --git a/man/BenchmarkResult.Rd b/man/BenchmarkResult.Rd index 65d778214..b8120f6e3 100644 --- a/man/BenchmarkResult.Rd +++ b/man/BenchmarkResult.Rd @@ -141,8 +141,8 @@ Set of (unique) hashes of all included \link{ResampleResult}s.} \item \href{#method-BenchmarkResult-format}{\code{BenchmarkResult$format()}} \item \href{#method-BenchmarkResult-print}{\code{BenchmarkResult$print()}} \item \href{#method-BenchmarkResult-combine}{\code{BenchmarkResult$combine()}} -\item \href{#method-BenchmarkResult-bundle}{\code{BenchmarkResult$bundle()}} -\item \href{#method-BenchmarkResult-unbundle}{\code{BenchmarkResult$unbundle()}} +\item \href{#method-BenchmarkResult-marshal}{\code{BenchmarkResult$marshal()}} +\item \href{#method-BenchmarkResult-unmarshal}{\code{BenchmarkResult$unmarshal()}} \item \href{#method-BenchmarkResult-score}{\code{BenchmarkResult$score()}} \item \href{#method-BenchmarkResult-aggregate}{\code{BenchmarkResult$aggregate()}} \item \href{#method-BenchmarkResult-filter}{\code{BenchmarkResult$filter()}} @@ -233,22 +233,22 @@ the object in its previous state. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-BenchmarkResult-bundle}{}}} -\subsection{Method \code{bundle()}}{ -Bundles all stored models. +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-BenchmarkResult-marshal}{}}} +\subsection{Method \code{marshal()}}{ +marshals all stored models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{BenchmarkResult$bundle()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{BenchmarkResult$marshal()}\if{html}{\out{
}} } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-BenchmarkResult-unbundle}{}}} -\subsection{Method \code{unbundle()}}{ -Unbundles all stored models. +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-BenchmarkResult-unmarshal}{}}} +\subsection{Method \code{unmarshal()}}{ +Unmarshals all stored models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{BenchmarkResult$unbundle()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{BenchmarkResult$unmarshal()}\if{html}{\out{
}} } } diff --git a/man/Learner.Rd b/man/Learner.Rd index 269bfe274..829e53b28 100644 --- a/man/Learner.Rd +++ b/man/Learner.Rd @@ -303,8 +303,8 @@ The following properties are currently standardized and understood by learners i \item \code{"importance"}: The learner supports extraction of importance scores, i.e. comes with an \verb{$importance()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). -\item \code{"bundle"}: To save learners with this property, you need to call \verb{$bundle()} first. -If a learner is in a bundled state, you call first need to call \verb{$unbundle()} to use it's model, e.g. for prediction. +\item \code{"marshal"}: To save learners with this property, you need to call \verb{$marshal()} first. +If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use it's model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/LearnerClassif.Rd b/man/LearnerClassif.Rd index c8994c993..4b2cbc2e5 100644 --- a/man/LearnerClassif.Rd +++ b/man/LearnerClassif.Rd @@ -139,8 +139,8 @@ The following properties are currently standardized and understood by learners i \item \code{"importance"}: The learner supports extraction of importance scores, i.e. comes with an \verb{$importance()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). -\item \code{"bundle"}: To save learners with this property, you need to call \verb{$bundle()} first. -If a learner is in a bundled state, you call first need to call \verb{$unbundle()} to use it's model, e.g. for prediction. +\item \code{"marshal"}: To save learners with this property, you need to call \verb{$marshal()} first. +If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use it's model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/LearnerRegr.Rd b/man/LearnerRegr.Rd index 759cbe094..77c07f8d5 100644 --- a/man/LearnerRegr.Rd +++ b/man/LearnerRegr.Rd @@ -129,8 +129,8 @@ The following properties are currently standardized and understood by learners i \item \code{"importance"}: The learner supports extraction of importance scores, i.e. comes with an \verb{$importance()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). -\item \code{"bundle"}: To save learners with this property, you need to call \verb{$bundle()} first. -If a learner is in a bundled state, you call first need to call \verb{$unbundle()} to use it's model, e.g. for prediction. +\item \code{"marshal"}: To save learners with this property, you need to call \verb{$marshal()} first. +If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use it's model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/ResampleResult.Rd b/man/ResampleResult.Rd index e01e08c2c..3e5e4e08d 100644 --- a/man/ResampleResult.Rd +++ b/man/ResampleResult.Rd @@ -107,8 +107,8 @@ Note that there can be multiple rows per resampling iteration if multiple errors \item \href{#method-ResampleResult-aggregate}{\code{ResampleResult$aggregate()}} \item \href{#method-ResampleResult-filter}{\code{ResampleResult$filter()}} \item \href{#method-ResampleResult-discard}{\code{ResampleResult$discard()}} -\item \href{#method-ResampleResult-bundle}{\code{ResampleResult$bundle()}} -\item \href{#method-ResampleResult-unbundle}{\code{ResampleResult$unbundle()}} +\item \href{#method-ResampleResult-marshal}{\code{ResampleResult$marshal()}} +\item \href{#method-ResampleResult-unmarshal}{\code{ResampleResult$unmarshal()}} \item \href{#method-ResampleResult-clone}{\code{ResampleResult$clone()}} } } @@ -351,22 +351,22 @@ the object in its previous state. } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-ResampleResult-bundle}{}}} -\subsection{Method \code{bundle()}}{ -Bundles all stored learner models. +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ResampleResult-marshal}{}}} +\subsection{Method \code{marshal()}}{ +marshals all stored learner models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResampleResult$bundle()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{ResampleResult$marshal()}\if{html}{\out{
}} } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-ResampleResult-unbundle}{}}} -\subsection{Method \code{unbundle()}}{ -Unbundles all stored learner models. +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ResampleResult-unmarshal}{}}} +\subsection{Method \code{unmarshal()}}{ +Unmarshals all stored learner models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResampleResult$unbundle()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{ResampleResult$unmarshal()}\if{html}{\out{
}} } } diff --git a/man/ResultData.Rd b/man/ResultData.Rd index 22ed59241..812d04f02 100644 --- a/man/ResultData.Rd +++ b/man/ResultData.Rd @@ -50,8 +50,8 @@ Returns \code{NULL} if the \link{ResultData} is empty.} \item \href{#method-ResultData-prediction}{\code{ResultData$prediction()}} \item \href{#method-ResultData-combine}{\code{ResultData$combine()}} \item \href{#method-ResultData-sweep}{\code{ResultData$sweep()}} -\item \href{#method-ResultData-bundle}{\code{ResultData$bundle()}} -\item \href{#method-ResultData-unbundle}{\code{ResultData$unbundle()}} +\item \href{#method-ResultData-marshal}{\code{ResultData$marshal()}} +\item \href{#method-ResultData-unmarshal}{\code{ResultData$unmarshal()}} \item \href{#method-ResultData-discard}{\code{ResultData$discard()}} \item \href{#method-ResultData-as_data_table}{\code{ResultData$as_data_table()}} \item \href{#method-ResultData-logs}{\code{ResultData$logs()}} @@ -334,22 +334,22 @@ Modified \code{self} (invisibly). } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-ResultData-bundle}{}}} -\subsection{Method \code{bundle()}}{ -Bundles all stored learner models. +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ResultData-marshal}{}}} +\subsection{Method \code{marshal()}}{ +Marshals all stored learner models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultData$bundle()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{ResultData$marshal()}\if{html}{\out{
}} } } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-ResultData-unbundle}{}}} -\subsection{Method \code{unbundle()}}{ -Unbundles all stored learner models. +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-ResultData-unmarshal}{}}} +\subsection{Method \code{unmarshal()}}{ +Unmarshals all stored learner models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultData$unbundle()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{ResultData$unmarshal()}\if{html}{\out{
}} } } diff --git a/man/benchmark.Rd b/man/benchmark.Rd index edb37a682..296aceb68 100644 --- a/man/benchmark.Rd +++ b/man/benchmark.Rd @@ -11,7 +11,7 @@ benchmark( encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling"), - bundle = TRUE + unmarshal = TRUE ) } \arguments{ @@ -59,9 +59,8 @@ providing a set with possible values \code{"task"}, \code{"learner"} and \code{"resampling"} for \link{Task}, \link{Learner} and \link{Resampling}, respectively. Per default, all input objects are cloned.} -\item{bundle}{(\code{logical(1)})\cr -Whether to bundle the learner(s) after the train-predict loop. -Default is \code{TRUE}.} +\item{unmarshal}{\code{\link{Learner}}\cr +Whether to unmarshal the learner.} } \value{ \link{BenchmarkResult}. diff --git a/man/bundling.Rd b/man/bundling.Rd deleted file mode 100644 index 18b8c336c..000000000 --- a/man/bundling.Rd +++ /dev/null @@ -1,60 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/bundle.R -\name{bundling} -\alias{bundling} -\alias{learner_unbundle} -\alias{learner_bundle} -\alias{learner_bundled} -\title{(Un)bundle a Learner} -\usage{ -learner_unbundle(learner) - -learner_bundle(learner) - -learner_bundled(learner) -} -\arguments{ -\item{learner}{\code{\link{Learner}}\cr -The learner to bundle.} -} -\description{ -Bundling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and -deserialized. The naming is inspired from \CRANpkg{bundle}. - -The function: -\itemize{ -\item \code{learner_bundle(learner)}: -Replaces the learner's model with the bundled model (in-place). -\item `learner_unbundle(learner) : -Replaces the learner's model with the unbundled model (in-place). -\item \code{learner_bundled(learner)}: -returns \code{FALSE} if the learner is either not trained or not bundled, otherwise \code{TRUE}. -Does not modify the learner. -} - -All three functions are primarily intended to be used when implementing bundling for a \code{\link{Learner}}. -Users who want to (un)bundle a \code{\link{Learner}} can instead use the public methods \verb{$bundle()} and \verb{$unbundle()} or the -field \verb{$bundled}, which is more in line with \code{mlr3}'s object oriented design. -} -\section{Implementing Bundling for a Learner}{ - -In order to implement bundling for a \code{\link{Learner}}, you need to add: -\itemize{ -\item the public methods \verb{$bundle()} and \verb{$unbundle()}, where you call \code{learner_bundle(self)} and -\code{learner_unbundle(self)} respectively. -\item the public method \verb{$bundle_model(model)}, which takes in a \code{\link{Learner}}'s model and returns it in bundled form, -without modifying the learner's state. Must not depend on the learner's state. -\item the public method \verb{$unbundle_model(model)}, which takes in a \code{\link{Learner}}'s bundled model and returns it in -unbundled form. Must not depend on the learner's state. -\item the active binding \verb{$bundled}, where you simply call \code{learner_bundled(self)}. -\item add the property \code{bundle} to the learner's properties. -} - -To test the bundling implementation, you can use the internal test helper \code{expect_bundleable()}. -This is also run in \code{expect_learner()} when a task is provided. - -For a concrete example on how to implement bundling, see the \code{LearnerTorch} class from -\href{https://github.com/mlr-org/mlr3torch}{mlr3torch}. -} - -\keyword{internal} diff --git a/man/marshalling.Rd b/man/marshalling.Rd new file mode 100644 index 000000000..98e642f6e --- /dev/null +++ b/man/marshalling.Rd @@ -0,0 +1,64 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/marshal.R +\name{marshalling} +\alias{marshalling} +\alias{learner_unmarshal} +\alias{learner_marshal} +\alias{learner_marshalled} +\alias{marshal_model} +\alias{unmarshal_model} +\alias{marshalled_model} +\title{(Un)marshal a Learner} +\usage{ +learner_unmarshal(learner) + +learner_marshal(learner) + +learner_marshalled(learner) + +marshal_model(model, ...) + +unmarshal_model(model, ...) + +marshalled_model(model) +} +\arguments{ +\item{learner}{\code{\link{Learner}}\cr +The learner to marshal.} +} +\description{ +Marshalling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and +deserialized. The naming is inspired from package \href{https://github.com/HenrikBengtsson/marshal}{marshall} and we +plan to fully migrate to this package once it is on CRAN. +The supported implementation until then should therfore be considered as a temporary solution and is likely +to change in the future. + +The central functions (and the only methods that are used by \code{mlr3} internally) are: +\itemize{ +\item the S3 generic \code{marshal_model(model, ...)}. +Which takes in a model and returns it in marshalled form. +The suffix \code{"_marshalled"} should be added to the class of the returned object and the root class must +be set to \code{"marshalled"}. +\item the S3 generic \code{unmarshal_model(model, ...)}. +Which takes in a model and returns it in unmarshalled form. +The returned object must not inherit from class \code{"marshalled"}. +\item the function \code{marshalled_model(model)}, which returns \code{TRUE} if the model inherits from class \code{"marshalled"} +and \code{FALSE} otherwise. +} + +In order to implement marshalling for a Learner, you only need to overload the \code{marshal_model} and \code{unmarshal_model} +methods and tag the learner with the \code{"marshal"} property accordingly. + +To make marshalling accessible in an R6-manner, you should also add the public methods \verb{$marshal()}, \verb{$unmarshal()} +and the active binding \verb{$marshalled}. +To make this as convenient as possible, the functions \code{learner_marshal(learner)}, \code{learner_unmarshal(learner)} +and \code{learner_marshalled(learner)} are provided and can be called from within the public methods. +All three functions throw an error if the learner is not trained and otherwise call +\code{marshal_model()}, \code{unmarshal_model()} or \code{marshalled_model()} on the learner's model. + +You can verify whether you have correctly implemented marshalling by using the internal test helper +\code{expect_marshallable_learner()}. This is also run by \code{expect_learner()} if a task is provided. + +For a concrete example on how to implement marshalling, see \code{\link{LearnerClassifLily}}. +} +\keyword{internal} diff --git a/man/mlr_learners_classif.lily.Rd b/man/mlr_learners_classif.lily.Rd new file mode 100644 index 000000000..376c308b9 --- /dev/null +++ b/man/mlr_learners_classif.lily.Rd @@ -0,0 +1,135 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/LearnerClassifLily.R +\name{mlr_learners_classif.lily} +\alias{mlr_learners_classif.lily} +\alias{LearnerClassifLily} +\title{Lily and Marshall} +\description{ +This learner is just like \code{\link{LearnerClassifDebug}}, but can be marshalled. +When the \code{count_marshalling} parameter is \code{TRUE}, the model contains a \code{marshal_count} that will be increased +by 1, each time \code{marshal_model} is called. +} +\section{Dictionary}{ + +This \link{Learner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_learners} or with the associated sugar function \code{\link[=lrn]{lrn()}}: + +\if{html}{\out{
}}\preformatted{mlr_learners$get("classif.lily") +lrn("classif.lily") +}\if{html}{\out{
}} +} + +\section{Meta Information}{ + +\itemize{ +\item Task type: \dQuote{classif} +\item Predict Types: \dQuote{response}, \dQuote{prob} +\item Feature Types: \dQuote{logical}, \dQuote{integer}, \dQuote{numeric}, \dQuote{character}, \dQuote{factor}, \dQuote{ordered} +\item Required Packages: \CRANpkg{mlr3} +} +} + +\section{Parameters}{ +\tabular{lllll}{ + Id \tab Type \tab Default \tab Levels \tab Range \cr + error_predict \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + error_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + message_predict \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + message_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + predict_missing \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + predict_missing_type \tab character \tab na \tab na, omit \tab - \cr + save_tasks \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr + segfault_predict \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + segfault_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + sleep_train \tab untyped \tab - \tab \tab - \cr + sleep_predict \tab untyped \tab - \tab \tab - \cr + threads \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr + warning_predict \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + warning_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + x \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + iter \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr + count_marshalling \tab logical \tab - \tab TRUE, FALSE \tab - \cr +} +} + +\section{Super classes}{ +\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerClassif]{mlr3::LearnerClassif}} -> \code{\link[mlr3:LearnerClassifDebug]{mlr3::LearnerClassifDebug}} -> \code{LearnerClassifLily} +} +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{marshalled}}{(logical(1))\cr +Whether the learner has been marshalled.} +} +\if{html}{\out{
}} +} +\section{Methods}{ +\subsection{Public methods}{ +\itemize{ +\item \href{#method-LearnerClassifLily-new}{\code{LearnerClassifLily$new()}} +\item \href{#method-LearnerClassifLily-marshal}{\code{LearnerClassifLily$marshal()}} +\item \href{#method-LearnerClassifLily-unmarshal}{\code{LearnerClassifLily$unmarshal()}} +\item \href{#method-LearnerClassifLily-clone}{\code{LearnerClassifLily$clone()}} +} +} +\if{html}{\out{ +
Inherited methods + +
+}} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerClassifLily-new}{}}} +\subsection{Method \code{new()}}{ +Creates a new instance of this \link[R6:R6Class]{R6} class. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerClassifLily$new()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerClassifLily-marshal}{}}} +\subsection{Method \code{marshal()}}{ +Marshals the learner. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerClassifLily$marshal()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerClassifLily-unmarshal}{}}} +\subsection{Method \code{unmarshal()}}{ +Unmarshal the learner. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerClassifLily$unmarshal()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerClassifLily-clone}{}}} +\subsection{Method \code{clone()}}{ +The objects of this class are cloneable with this method. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerClassifLily$clone(deep = FALSE)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{deep}}{Whether to make a deep clone.} +} +\if{html}{\out{
}} +} +} +} diff --git a/man/resample.Rd b/man/resample.Rd index 935c6a97c..17d1ad850 100644 --- a/man/resample.Rd +++ b/man/resample.Rd @@ -13,7 +13,7 @@ resample( encapsulate = NA_character_, allow_hotstart = FALSE, clone = c("task", "learner", "resampling"), - bundle = TRUE + unmarshal = TRUE ) } \arguments{ @@ -60,9 +60,8 @@ providing a set with possible values \code{"task"}, \code{"learner"} and \code{"resampling"} for \link{Task}, \link{Learner} and \link{Resampling}, respectively. Per default, all input objects are cloned.} -\item{bundle}{(\code{logical(1)})\cr -Whether to bundle the learner(s) after the train-predict loop. -Default is \code{TRUE}.} +\item{unmarshal}{\code{\link{Learner}}\cr +Whether to unmarshal the learner.} } \value{ \link{ResampleResult}. diff --git a/tests/testthat/test_HotstartStack.R b/tests/testthat/test_HotstartStack.R index 4abcfc16d..a074f5e9d 100644 --- a/tests/testthat/test_HotstartStack.R +++ b/tests/testthat/test_HotstartStack.R @@ -409,3 +409,11 @@ test_that("HotstartStack threshold works", { hot$add(learner_2) expect_data_table(hot$stack, nrows = 1) }) + +test_that("error when adding marshalled learner", { + hot = HotstartStack$new() + learner = lrn("classif.lily") + learner$train(tsk("iris")) + learner$marshal() + expect_error(hot$add(learner), "unmarshalled") +}) diff --git a/tests/testthat/test_Learner.R b/tests/testthat/test_Learner.R index bdd5f709d..769a6c86f 100644 --- a/tests/testthat/test_Learner.R +++ b/tests/testthat/test_Learner.R @@ -324,3 +324,20 @@ test_that("Models can be replaced", { learner$model$location = 1 expect_equal(learner$model$location, 1) }) + +test_that("marshalling and encapsulation", { + task = tsk("iris") + learner = lrn("classif.lily", count_marshalling = TRUE) + + # callr encapsulation causes marshalling + learner$encapsulate = c(train = "callr") + learner$train(task) + expect_equal(learner$model$marshal_count, 1) + expect_false(learner$marshalled) + expect_prediction(learner$predict(task)) + + # no marshalling with no other encapsulation + learner$encapsulate = c(train = "none") + learner$train(task) + expect_equal(learner$model$marshal_count, 0) +}) diff --git a/tests/testthat/test_LearnerClassifLily.R b/tests/testthat/test_LearnerClassifLily.R new file mode 100644 index 000000000..57691b090 --- /dev/null +++ b/tests/testthat/test_LearnerClassifLily.R @@ -0,0 +1,34 @@ +test_that("lily", { + task = tsk("iris") + learner = lrn("classif.lily") + learner$train(task) + expect_false(learner$marshalled) + learner$marshal() + expect_true(learner$marshalled) + expect_error(learner$predict(task), "has not been unmarshalled") + learner$unmarshal() + expect_learner(learner, task) +}) + +test_that("marshal count works", { + # to mock that marshalling behaves as expected, we need to be know how often it happened + # note that this means that marshal_model modifies the model in a permanent way, i.e. this is not reversed by + # unmarshal_model. + learner = lrn("classif.lily", count_marshalling = TRUE) + task = tsk("iris") + learner$train(task) + expect_equal(learner$model$marshal_count, 0) + learner$marshal()$unmarshal() + expect_equal(learner$model$marshal_count, 1) + learner$marshal()$unmarshal() + expect_equal(learner$model$marshal_count, 2) + + # TO make the lily learner more realistic (i.e. (un)marshalling leaves the object unchanged) + # the count_marshalling parameter can also be set to FALSE + learner2 = lrn("classif.lily", count_marshalling = FALSE) + learner2$train(task) + expect_true(is.null(learner2$model$marshal_count)) + model1 = learner2$model + model2 = learner2$marshal()$unmarshal()$model + expect_equal(model1, model2) +}) diff --git a/tests/testthat/test_Measure.R b/tests/testthat/test_Measure.R index 312f10711..fcae25fa2 100644 --- a/tests/testthat/test_Measure.R +++ b/tests/testthat/test_Measure.R @@ -130,3 +130,13 @@ test_that("time_train is > 0", { res = rr$score(msr("time_train")) expect_gte(res$time_train, 0) }) + +test_that("scoring fails when measure requires_model, but model is in marshalled state", { + measure = msr("classif.acc") + measure$properties = c(measure$properties, "requires_model") + + rr = resample(tsk("iris"), lrn("classif.lily"), rsmp("holdout")) + rr$marshal() + + rr$score(msr("selected_features")) +}) diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index 72cff51c3..1ef68c7b2 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -478,89 +478,68 @@ test_that("param_values in benchmark", { }) -test_that("bundling", { - task = tsk("mtcars") - LearnerRegrTest = R6Class("LearnerRegrTest", - inherit = LearnerRegrFeatureless, - public = list( - initialize = function(class = "bundle") { - private$.class = class - super$initialize() - self$properties = c("bundle", self$properties) - }, - bundle = function() { - learner_bundle(self) - }, - unbundle = function() { - learner_unbundle(self) - }, - bundle_model = function(model) { - structure(list(model), class = private$.class) - }, - unbundle_model = function(model) { - model[[1L]] - } - ), - active = list( - bundled = function() { - learner_bundled(self) - } - ), - private = list( - .class = NULL - ) - ) - learner = LearnerRegrTest$new() - resampling = rsmp("holdout")$instantiate(task) - - # Learner can be bundled during benchmark() - bmr1 = benchmark(benchmark_grid(task, learner, resampling), store_models = TRUE, bundle = TRUE) - lrn_rec = bmr1$resample_results$resample_result[[1]]$learners[[1]] - expect_true(lrn_rec$bundled) - - # learner can be unbundled after benchmark() - lrn_rec$unbundle() - expect_false(lrn_rec$bundled) - - # result is the same with and without bundling - bmr2 = benchmark(benchmark_grid(task, lrn("regr.featureless"), resampling), store_models = TRUE, bundle = TRUE) - expect_equal(as.data.table(bmr1$aggregate())$regr.mse, as.data.table(bmr2$aggregate())$regr.mse) - - # bundling can be disabled - bmr3 = benchmark(benchmark_grid(task, learner, resampling), store_models = TRUE, bundle = FALSE) - expect_false(bmr3$resample_results$resample_result[[1]]$learners[[1]]$bundled) - - # can (un)bundle benchmark result - # we are mixing differnt bundling methods - l1 = LearnerRegrTest$new(class = "a") - l2 = lrn("regr.rpart") - l3 = LearnerRegrTest$new(class = "b") - # to give l1 and l3 different hashes - class(l3) = c("LearenrRegrTest2", class(l3)[-1]) - expect_false(identical(l1$phash, l3$phash)) - - design = benchmark_grid(tsk("mtcars"), list(l1, l2, l3), rsmp("cv", folds = 3)) - bmr = benchmark(design, store_models = TRUE) - - bmr$unbundle() - - walk(bmr$resample_result(1)$learners, function(l) { - expect_class(l$model, "regr.featureless_model") - }) - walk(bmr$resample_result(2)$learners, function(l) { - expect_class(l$model, "rpart") - }) - walk(bmr$resample_result(3)$learners, function(l) { - expect_class(l$model, "regr.featureless_model") +test_that("parallel execution automatically triggers marshalling", { + learner = lrn("classif.lily", count_marshalling = TRUE) + task = tsk("iris") + resampling = rsmp("holdout") + design = benchmark_grid(task, learner, resampling) + bmr = with_future(future::multisession, { + benchmark(design, store_models = TRUE, unmarshal = TRUE) }) - bmr$bundle() - walk(bmr$resample_result(1)$learners, function(l) { - expect_class(l$model, "a") + expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 1) + expect_false(bmr$resample_result(1)$learners[[1]]$marshalled) +}) + +test_that("sequential execution does not trigger marshalling", { + learner = lrn("classif.lily", count_marshalling = TRUE) + task = tsk("iris") + resampling = rsmp("holdout") + design = benchmark_grid(task, learner, resampling) + bmr = with_future(future::sequential, { + benchmark(design, store_models = TRUE, unmarshal = TRUE) }) - walk(bmr$resample_result(2)$learners, function(l) { - expect_class(l$model, "rpart") + expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 0) +}) + +test_that("parallel exeuction and callr marshall twice", { + learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) + task = tsk("iris") + resampling = rsmp("holdout") + design = benchmark_grid(task, learner, resampling) + bmr = with_future(future::multisession, { + benchmark(design, store_models = TRUE, unmarshal = TRUE) }) - walk(bmr$resample_result(3)$learners, function(l) { - expect_class(l$model, "b") + expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 2) + expect_false(bmr$resample_result(1)$learners[[1]]$marshalled) +}) + + +test_that("unmarshal parameter is respected", { + learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) + task = tsk("iris") + resampling = rsmp("holdout") + design = benchmark_grid(task, learner, resampling) + bmr = with_future(future::multisession, { + list( + marshalled = benchmark(design, store_models = TRUE, unmarshal = FALSE), + unmarshalled = benchmark(design, store_models = TRUE, unmarshal = TRUE) + ) }) + expect_false(bmr$unmarshalled$resample_result(1)$learners[[1]]$marshalled) + expect_true(bmr$marshalled$resample_result(1)$learners[[1]]$marshalled) +}) + +test_that("BenchmarkResult can be (un)marshalled", { + bmr = benchmark(benchmark_grid(tsk("iris"), lrn("classif.lily"), rsmp("holdout")), store_models = TRUE) + expect_false(bmr$resample_result(1)$learners[[1]]$marshalled) + bmr$marshal() + expect_true(bmr$resample_result(1)$learners[[1]]$marshalled) + bmr$unmarshal() + expect_false(bmr$resample_result(1)$learners[[1]]$marshalled) + + # also works with non-marshallable learner + bmr1 = benchmark(benchmark_grid(tsk("iris"), lrn("classif.featureless"), rsmp("holdout")), store_models = TRUE) + model = bmr1$resample_result(1)$learners[[1]]$model + bmr1$unmarshal() + expect_equal(bmr1$resample_result(1)$learners[[1]]$model, model) }) diff --git a/tests/testthat/test_bundle.R b/tests/testthat/test_bundle.R deleted file mode 100644 index 11aa72a8c..000000000 --- a/tests/testthat/test_bundle.R +++ /dev/null @@ -1,83 +0,0 @@ -test_that("bundleable learner behaves as expected", { - task = tsk("mtcars") - LearnerRegrTest = R6Class("LearnerRegrTest", - inherit = LearnerRegrFeatureless, - public = list( - initialize = function() { - super$initialize() - self$id = "regr.test" - self$properties = c("bundle", self$properties) - }, - bundle = function() { - learner_bundle(self) - }, - unbundle = function() { - learner_unbundle(self) - }, - bundle_model = function(model) { - private$.tmp_model = model - "bundle" - }, - unbundle_model = function(model) { - model = private$.tmp_model - private$.tmp_model = NULL - private$.counter = private$.counter + 1 - model - } - ), - private = list( - .tmp_model = NULL, - .counter = 0 - ), - active = list( - bundled = function() { - learner_bundled(self) - } - ) - ) - learner = LearnerRegrTest$new() - # bundleable learner - expect_true("bundle" %in% learner$properties) - expect_bundleable(learner, task) - - # callr encapsulation causes bundling - learner2 = LearnerRegrTest$new() - learner2$encapsulate = c(train = "callr") - learner2$train(task) - - learner3 = LearnerRegrTest$new() - learner3$encapsulate = c(train = "try") - learner3$train(task) - expect_false(learner3$bundled) - - # for callr, we had to unbundle - expect_equal(get_private(learner2)$.counter, 1) - # for other encapsulation, no need to unbundle becausse it was not bundled - expect_equal(get_private(learner3)$.counter, 0) - - learner4 = LearnerRegrTest$new() - learner4$train(task) - # invisible - expect_invisible(learner$bundle()) - expect_invisible(learner$unbundle()) - - # bundling ResampleResult without stored models does not break state - rr = resample(task, learner4, rsmp("cv"), store_models = FALSE) - s1 = map(rr$learners, function(x) x$state) - rr$bundle() - s2 = map(rr$learners, function(x) x$state) - expect_equal(s1, s2) - - # bundles is FALSE when model was nulled - learner5 = LearnerRegrTest$new() - learner5$train(task) - learner5$bundle() - learner5$model = NULL - expect_false(learner5$bundled) -}) - -test_that("unbundleable learner's bundle is NULL", { - learner = lrn("regr.featureless") - learner$train(tsk("mtcars")) - expect_true(is.null(learner$state$bundled)) -}) diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R new file mode 100644 index 000000000..91a42d7c0 --- /dev/null +++ b/tests/testthat/test_marshal.R @@ -0,0 +1,18 @@ +test_that("marshal works as expectected", { + learner = lrn("classif.lily") + task = tsk("iris") + expect_error(learner_marshal(learner), "not been trained") + expect_error(learner_unmarshal(learner), "not been trained") + expect_error(learner_marshalled(learner), "not been trained") + + learner$train(task) + expect_false(learner_marshalled(learner)) + learner$marshal() + expect_true(learner_marshalled(learner)) + learner$unmarshal() + expect_false(learner_marshalled(learner)) + + # default does nothing + expect_equal(marshal_model(1), 1) + expect_equal(unmarshal_model(1), 1) +}) diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index d0b4cc1ee..1e9100cbe 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -157,74 +157,64 @@ test_that("as_resample_result works for result data", { expect_class(rr2, "ResampleResult") }) -test_that("bundling", { - task = tsk("mtcars") - LearnerRegrTest = R6Class("LearnerRegrTest", - inherit = LearnerRegrFeatureless, - public = list( - initialize = function() { - super$initialize() - self$properties = c("bundle", self$properties) - }, - bundle = function() learner_bundle(self), - unbundle = function() learner_unbundle(self), - bundle_model = function(model) { - structure(list(model), class = "bundled") - }, - unbundle_model = function(model) { - model[[1L]] - } - ), - active = list( - bundled = function() learner_bundled(self) - ), - private = list( - .tmp_model = NULL +test_that("parallel execution automatically triggers marshalling", { + learner = lrn("classif.lily", count_marshalling = TRUE) + task = tsk("iris") + resampling = rsmp("holdout") + rr = with_future(future::multisession, { + resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) + }) + expect_equal(rr$learners[[1]]$model$marshal_count, 1) + expect_false(rr$learners[[1]]$marshalled) +}) + +test_that("sequential execution does not trigger marshalling", { + learner = lrn("classif.lily", count_marshalling = TRUE) + task = tsk("iris") + resampling = rsmp("holdout") + rr = with_future(future::sequential, { + resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) + }) + expect_equal(rr$learners[[1]]$model$marshal_count, 0) +}) + +test_that("parallel exeuction and callr marshall twice", { + learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) + task = tsk("iris") + resampling = rsmp("holdout") + rr = with_future(future::multisession, { + resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) + }) + expect_equal(rr$learners[[1]]$model$marshal_count, 2) + expect_false(rr$learners[[1]]$marshalled) +}) + + +test_that("unmarshal parameter is respected", { + learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) + task = tsk("iris") + resampling = rsmp("holdout") + rr = with_future(future::multisession, { + list( + marshalled = resample(task, learner, resampling, store_models = TRUE, unmarshal = FALSE), + unmarshalled = resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) ) - ) - learner = LearnerRegrTest$new() - - # allow to bundle during resample() - resampling = rsmp("cv", folds = 2L) - resampling$instantiate(task) # to compare different resample results - rr1 = resample(task, learner, resampling, store_models = TRUE, bundle = TRUE) - lrn_rec = rr1$learners[[1L]] - expect_true(lrn_rec$bundled) - expect_false(lrn_rec$unbundle()$bundled) - - # ResampleResult can be unbundled. - expect_true(rr1$learners[[1]]$bundled) - expect_class(rr1$learners[[1]]$model, "bundled") - expect_true(rr1$learners[[2]]$bundled) - expect_class(rr1$learners[[2]]$model, "bundled") - - rr1$unbundle() - expect_false(rr1$learners[[1]]$bundled) - expect_class(rr1$learners[[1]]$model, "regr.featureless_model") - expect_false(rr1$learners[[2]]$bundled) - expect_class(rr1$learners[[2]]$model, "regr.featureless_model") - - rr1$bundle() - expect_true(rr1$learners[[1]]$bundled) - expect_class(rr1$learners[[1]]$model, "bundled") - expect_true(rr1$learners[[2]]$bundled) - expect_class(rr1$learners[[2]]$model, "bundled") - - # bundled resamples are equivalent to unbundled results - rr2 = resample(task, lrn("regr.featureless"), resampling, store_models = TRUE) - rr3 = resample(task, learner, resampling, store_models = FALSE) - expect_equal(rr1$aggregate(), rr3$aggregate()) - expect_equal(rr1$aggregate(), rr2$aggregate()) - - # bundling can be disabled - rr4 = resample(task, learner, resampling, bundle = FALSE, store_models = TRUE) - expect_false(rr4$learners[[1]]$bundled) -}) - -test_that("bundling does nothing with unbundleable learners", { - rr = resample(tsk("mtcars"), lrn("regr.featureless"), rsmp("cv", folds = 2)) - rr$unbundle() - expect_resample_result(rr) - rr$bundle() - expect_resample_result(rr) + }) + expect_false(rr$unmarshalled$learners[[1]]$marshalled) + expect_true(rr$marshalled$learners[[1]]$marshalled) +}) + +test_that("ResampleResult can be (un)marshalled", { + rr = resample(tsk("iris"), lrn("classif.lily"), rsmp("holdout"), store_models = TRUE) + expect_false(rr$learners[[1]]$marshalled) + rr$marshal() + expect_true(rr$learners[[1]]$marshalled) + rr$unmarshal() + expect_false(rr$learners[[1]]$marshalled) + + # also works with non-marshallable learner + rr1 = resample(tsk("iris"), lrn("classif.featureless"), rsmp("holdout"), store_models = TRUE) + model = rr1$learners[[1]]$model + rr1$unmarshal() + expect_equal(rr1$learners[[1]]$model, model) }) From f98424b24e889bf45f44a3a3f7cba5534abc16a3 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Thu, 1 Feb 2024 19:37:26 +0100 Subject: [PATCH 13/47] Update R/Measure.R --- R/Measure.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/Measure.R b/R/Measure.R index 2169bf8ce..f779e9d11 100644 --- a/R/Measure.R +++ b/R/Measure.R @@ -187,7 +187,7 @@ Measure = R6Class("Measure", stopf("Measure '%s' requires the trained model", self$id) } if ("requires_model" %in% self$properties && marshalled_model(learner$model)) { - stopf("Measure '%s' requires the trained model, but model is un marshalled form", self$id) + stopf("Measure '%s' requires the trained model, but model is marshalled form", self$id) } if ("requires_train_set" %in% self$properties && is.null(train_set)) { From 50919d28f68965eb604e43b444dfd0113450b9f3 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 14 Feb 2024 14:08:23 +0100 Subject: [PATCH 14/47] Update man-roxygen/param_learner_properties.R --- man-roxygen/param_learner_properties.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man-roxygen/param_learner_properties.R b/man-roxygen/param_learner_properties.R index e02bcf946..df51e66a8 100644 --- a/man-roxygen/param_learner_properties.R +++ b/man-roxygen/param_learner_properties.R @@ -8,4 +8,4 @@ #' * `"selected_features"`: The learner supports extraction of the set of selected features, i.e. comes with a `$selected_features()` extractor function (see section on optional extractors in [Learner]). #' * `"oob_error"`: The learner supports extraction of estimated out of bag error, i.e. comes with a `oob_error()` extractor function (see section on optional extractors in [Learner]). #' * `"marshal"`: To save learners with this property, you need to call `$marshal()` first. -#' If a learner is in a marshalled state, you call first need to call `$unmarshal()` to use it's model, e.g. for prediction. +#' If a learner is in a marshalled state, you call first need to call `$unmarshal()` to use its model, e.g. for prediction. From 77af5eefac1f99e147caafafea87a130223bb623 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 20 Feb 2024 16:23:14 +0100 Subject: [PATCH 15/47] better marshal behavior --- DESCRIPTION | 2 +- NAMESPACE | 3 ++ R/ResultData.R | 6 ++-- R/marshal.R | 37 ++++++++++++++++++++++--- R/worker.R | 14 ++++++---- man/BenchmarkResult.Rd | 4 +-- man/DataBackendDataTable.Rd | 2 +- man/DataBackendMatrix.Rd | 2 +- man/Learner.Rd | 4 +-- man/LearnerClassif.Rd | 6 ++-- man/LearnerRegr.Rd | 6 ++-- man/Measure.Rd | 4 +-- man/MeasureClassif.Rd | 6 ++-- man/MeasureRegr.Rd | 6 ++-- man/MeasureSimilarity.Rd | 6 ++-- man/PredictionClassif.Rd | 4 +-- man/PredictionRegr.Rd | 4 +-- man/Resampling.Rd | 6 ++-- man/ResultData.Rd | 2 ++ man/Task.Rd | 4 +-- man/TaskClassif.Rd | 6 ++-- man/TaskGenerator.Rd | 4 +-- man/TaskRegr.Rd | 6 ++-- man/TaskSupervised.Rd | 6 ++-- man/TaskUnsupervised.Rd | 6 ++-- man/as_data_backend.Rd | 4 +-- man/mlr_learners.Rd | 2 +- man/mlr_learners_classif.debug.Rd | 6 ++-- man/mlr_learners_classif.featureless.Rd | 6 ++-- man/mlr_learners_classif.rpart.Rd | 6 ++-- man/mlr_learners_regr.debug.Rd | 6 ++-- man/mlr_learners_regr.featureless.Rd | 6 ++-- man/mlr_learners_regr.rpart.Rd | 6 ++-- man/mlr_measures.Rd | 2 +- man/mlr_measures_aic.Rd | 6 ++-- man/mlr_measures_bic.Rd | 6 ++-- man/mlr_measures_classif.acc.Rd | 10 +++---- man/mlr_measures_classif.auc.Rd | 20 ++++++------- man/mlr_measures_classif.bacc.Rd | 10 +++---- man/mlr_measures_classif.bbrier.Rd | 20 ++++++------- man/mlr_measures_classif.ce.Rd | 10 +++---- man/mlr_measures_classif.costs.Rd | 16 +++++------ man/mlr_measures_classif.dor.Rd | 20 ++++++------- man/mlr_measures_classif.fbeta.Rd | 20 ++++++------- man/mlr_measures_classif.fdr.Rd | 20 ++++++------- man/mlr_measures_classif.fn.Rd | 16 +++++------ man/mlr_measures_classif.fnr.Rd | 16 +++++------ man/mlr_measures_classif.fomr.Rd | 20 ++++++------- man/mlr_measures_classif.fp.Rd | 16 +++++------ man/mlr_measures_classif.fpr.Rd | 16 +++++------ man/mlr_measures_classif.logloss.Rd | 10 +++---- man/mlr_measures_classif.mauc_au1p.Rd | 10 +++---- man/mlr_measures_classif.mauc_au1u.Rd | 10 +++---- man/mlr_measures_classif.mauc_aunp.Rd | 10 +++---- man/mlr_measures_classif.mauc_aunu.Rd | 10 +++---- man/mlr_measures_classif.mbrier.Rd | 10 +++---- man/mlr_measures_classif.mcc.Rd | 20 ++++++------- man/mlr_measures_classif.npv.Rd | 20 ++++++------- man/mlr_measures_classif.ppv.Rd | 20 ++++++------- man/mlr_measures_classif.prauc.Rd | 20 ++++++------- man/mlr_measures_classif.precision.Rd | 20 ++++++------- man/mlr_measures_classif.recall.Rd | 20 ++++++------- man/mlr_measures_classif.sensitivity.Rd | 20 ++++++------- man/mlr_measures_classif.specificity.Rd | 20 ++++++------- man/mlr_measures_classif.tn.Rd | 16 +++++------ man/mlr_measures_classif.tnr.Rd | 16 +++++------ man/mlr_measures_classif.tp.Rd | 12 ++++---- man/mlr_measures_classif.tpr.Rd | 12 ++++---- man/mlr_measures_debug_classif.Rd | 6 ++-- man/mlr_measures_elapsed_time.Rd | 6 ++-- man/mlr_measures_oob_error.Rd | 6 ++-- man/mlr_measures_selected_features.Rd | 6 ++-- man/mlr_resamplings.Rd | 2 +- man/mlr_resamplings_bootstrap.Rd | 6 ++-- man/mlr_resamplings_custom.Rd | 4 +-- man/mlr_resamplings_custom_cv.Rd | 4 +-- man/mlr_resamplings_cv.Rd | 6 ++-- man/mlr_resamplings_holdout.Rd | 6 ++-- man/mlr_resamplings_insample.Rd | 6 ++-- man/mlr_resamplings_loo.Rd | 6 ++-- man/mlr_resamplings_repeated_cv.Rd | 6 ++-- man/mlr_resamplings_subsampling.Rd | 6 ++-- man/mlr_task_generators_2dnormals.Rd | 4 +-- man/mlr_task_generators_cassini.Rd | 4 +-- man/mlr_task_generators_circle.Rd | 4 +-- man/mlr_task_generators_friedman1.Rd | 4 +-- man/mlr_task_generators_moons.Rd | 4 +-- man/mlr_task_generators_simplex.Rd | 4 +-- man/mlr_task_generators_smiley.Rd | 4 +-- man/mlr_task_generators_spirals.Rd | 4 +-- man/mlr_task_generators_xor.Rd | 4 +-- man/mlr_tasks.Rd | 2 +- man/mlr_tasks_boston_housing.Rd | 6 ++-- man/mlr_tasks_breast_cancer.Rd | 6 ++-- man/mlr_tasks_german_credit.Rd | 6 ++-- man/mlr_tasks_iris.Rd | 6 ++-- man/mlr_tasks_mtcars.Rd | 6 ++-- man/mlr_tasks_penguins.Rd | 6 ++-- man/mlr_tasks_pima.Rd | 6 ++-- man/mlr_tasks_sonar.Rd | 6 ++-- man/mlr_tasks_spam.Rd | 6 ++-- man/mlr_tasks_wine.Rd | 6 ++-- man/mlr_tasks_zoo.Rd | 6 ++-- tests/testthat/test_benchmark.R | 2 +- tests/testthat/test_marshal.R | 29 ++++++++++++++++--- tests/testthat/test_resample.R | 2 +- 106 files changed, 496 insertions(+), 435 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 23e4c77ca..8605d7aec 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -74,7 +74,7 @@ Config/testthat/edition: 3 Config/testthat/parallel: false NeedsCompilation: no Roxygen: list(markdown = TRUE, r6 = TRUE) -RoxygenNote: 7.2.3.9000 +RoxygenNote: 7.3.1 Collate: 'mlr_reflections.R' 'BenchmarkResult.R' diff --git a/NAMESPACE b/NAMESPACE index 43c14bfca..72e19638b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -87,8 +87,10 @@ S3method(fix_factor_levels,data.table) S3method(head,Task) S3method(is_missing_prediction_data,PredictionDataClassif) S3method(is_missing_prediction_data,PredictionDataRegr) +S3method(marshal_model,"NULL") S3method(marshal_model,classif_lily_model) S3method(marshal_model,default) +S3method(marshal_model,marshalled) S3method(partition,Task) S3method(partition,TaskClassif) S3method(partition,TaskRegr) @@ -108,6 +110,7 @@ S3method(summary,Task) S3method(tail,Task) S3method(unmarshal_model,classif_lily_model_marshalled) S3method(unmarshal_model,default) +S3method(unmarshal_model,marshalled) export(BenchmarkResult) export(DataBackend) export(DataBackendDataTable) diff --git a/R/ResultData.R b/R/ResultData.R index f3d7c6072..1c1c18f9e 100644 --- a/R/ResultData.R +++ b/R/ResultData.R @@ -244,16 +244,18 @@ ResultData = R6Class("ResultData", #' @description #' Marshals all stored learner models. + #' This will do nothing to models that are already marshalled. marshal = function() { learner_state = NULL - self$data$fact[, learner_state := lapply(learner_state, marshal_state)] + self$data$fact[, learner_state := lapply(learner_state, marshal_state_if_model)] invisible(self) }, #' @description #' Unmarshals all stored learner models. + #' This will do nothing to models which are not marshalled. unmarshal = function() { learner_state = NULL - self$data$fact[, learner_state := lapply(learner_state, unmarshal_state)] + self$data$fact[, learner_state := lapply(learner_state, unmarshal_state_if_model)] invisible(self) }, diff --git a/R/marshal.R b/R/marshal.R index 38625dc7e..6dd6d8986 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -40,9 +40,11 @@ #' @keywords internal #' @export learner_unmarshal = function(learner) { + # no need to check for 'marshal' property as this method should only be available for such learners if (is.null(learner$model)) { stopf("Cannot unmarshal, Learner '%s' has not been trained yet", learner$id) } + # this will do nothing if the model was not marshalled learner$model = unmarshal_model(learner$model) invisible(learner) } @@ -50,9 +52,11 @@ learner_unmarshal = function(learner) { #' @rdname marshalling #' @export learner_marshal = function(learner) { + # no need to check for 'marshal' property as this method should only be available for such learners if (is.null(learner$model)) { stopf("Cannot marshal, Learner '%s' has not been trained yet", learner$id) } + # this will do nothing if the model was already marshalled learner$model = marshal_model(learner$model) invisible(learner) } @@ -60,6 +64,7 @@ learner_marshal = function(learner) { #' @rdname marshalling #' @export learner_marshalled = function(learner) { + # no need to check for 'marshal' property as this method should only be available for such learners if (is.null(learner$model)) { stopf("Cannot check marshalled status, Learner '%s' has not been trained yet", learner$id) } @@ -86,6 +91,26 @@ marshalled_model = function(model) { #' @export marshal_model.default = function(model, ...) { + classes = class(model) + class(model) = c(paste0(classes, "_marshalled"), "marshalled") + model +} + +#' @export +marshal_model.marshalled = function(model, ...) { + model +} + +#' @export +marshal_model.NULL = function(model, ...) { + # NULL cannot have a class, so it is not covered by the default method + model +} + +#' @export +unmarshal_model.marshalled = function(model, ...) { + classes = class(model)[-length(class(model))] + class(model) = gsub("_marshalled$", "", classes) model } @@ -94,12 +119,16 @@ unmarshal_model.default = function(model, ...) { model } -marshal_state = function(state, ...) { - state$model = marshal_model(state$model) +marshal_state_if_model = function(state, ...) { + if (!is.null(state$model)) { + state$model = marshal_model(state$model) + } state } -unmarshal_state = function(state, ...) { - state$model = unmarshal_model(state$model) +unmarshal_state_if_model = function(state, ...) { + if (!is.null(state$model)) { + state$model = unmarshal_model(state$model) + } state } diff --git a/R/worker.R b/R/worker.R index f1bf25ca0..0de22d6d1 100644 --- a/R/worker.R +++ b/R/worker.R @@ -18,8 +18,7 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL stopf("Learner '%s' on task '%s' returned NULL during internal %s()", learner$id, task$id, mode) } - if (learner$encapsulate[["train"]] == "callr") { - # the default method of marshal_model does nothing + if (learner$encapsulate[["train"]] == "callr" && "marshal" %in% learner$properties) { model = marshal_model(model) } @@ -74,11 +73,16 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL train_time = result$elapsed # unmarshal_model does nothing if the model was not marshalled + + # We always want to unmarshal the model, because either: # a) the user called $train() manually or # b) we are within worker and still have to make a prediction + if ("marshal" %in% learner$properties) { + result$result = unmarshal_model(result$result) + } learner$state = insert_named(learner$state, list( - model = unmarshal_model(result$result), + model = result$result, log = log, train_time = train_time, param_vals = learner$param_set$values, @@ -275,8 +279,8 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, if (!store_models) { lg$debug("Erasing stored model for learner '%s'", learner$id) learner$state$model = NULL - } else if (!is_sequential) { - lg$debug("Marshalling model for learner '%s' because of non-sequential execution", learner$id) + } else if ("marshal" %in% learner$properties && !is_sequential) { + lg$debug("Marshalling model for learner '%s'", learner$id) learner$model = marshal_model(learner$model) } diff --git a/man/BenchmarkResult.Rd b/man/BenchmarkResult.Rd index b8120f6e3..41d42d1a9 100644 --- a/man/BenchmarkResult.Rd +++ b/man/BenchmarkResult.Rd @@ -77,8 +77,8 @@ print(bmr) } Other benchmark: -\code{\link{benchmark_grid}()}, -\code{\link{benchmark}()} +\code{\link{benchmark}()}, +\code{\link{benchmark_grid}()} } \concept{benchmark} \section{Active bindings}{ diff --git a/man/DataBackendDataTable.Rd b/man/DataBackendDataTable.Rd index a1ef4f2f2..3e7310d4d 100644 --- a/man/DataBackendDataTable.Rd +++ b/man/DataBackendDataTable.Rd @@ -32,8 +32,8 @@ e.g. SQL servers or \CRANpkg{duckdb}. } Other DataBackend: -\code{\link{DataBackendMatrix}}, \code{\link{DataBackend}}, +\code{\link{DataBackendMatrix}}, \code{\link{as_data_backend.Matrix}()} } \concept{DataBackend} diff --git a/man/DataBackendMatrix.Rd b/man/DataBackendMatrix.Rd index cb8980e91..52faf518b 100644 --- a/man/DataBackendMatrix.Rd +++ b/man/DataBackendMatrix.Rd @@ -34,8 +34,8 @@ e.g. SQL servers or \CRANpkg{duckdb}. } Other DataBackend: -\code{\link{DataBackendDataTable}}, \code{\link{DataBackend}}, +\code{\link{DataBackendDataTable}}, \code{\link{as_data_backend.Matrix}()} } \concept{DataBackend} diff --git a/man/Learner.Rd b/man/Learner.Rd index 829e53b28..697501794 100644 --- a/man/Learner.Rd +++ b/man/Learner.Rd @@ -98,13 +98,13 @@ for established default tuning spaces. Other Learner: \code{\link{LearnerClassif}}, \code{\link{LearnerRegr}}, +\code{\link{mlr_learners}}, \code{\link{mlr_learners_classif.debug}}, \code{\link{mlr_learners_classif.featureless}}, \code{\link{mlr_learners_classif.rpart}}, \code{\link{mlr_learners_regr.debug}}, \code{\link{mlr_learners_regr.featureless}}, -\code{\link{mlr_learners_regr.rpart}}, -\code{\link{mlr_learners}} +\code{\link{mlr_learners_regr.rpart}} } \concept{Learner} \section{Public fields}{ diff --git a/man/LearnerClassif.Rd b/man/LearnerClassif.Rd index 4b2cbc2e5..478347768 100644 --- a/man/LearnerClassif.Rd +++ b/man/LearnerClassif.Rd @@ -60,15 +60,15 @@ for established default tuning spaces. } Other Learner: -\code{\link{LearnerRegr}}, \code{\link{Learner}}, +\code{\link{LearnerRegr}}, +\code{\link{mlr_learners}}, \code{\link{mlr_learners_classif.debug}}, \code{\link{mlr_learners_classif.featureless}}, \code{\link{mlr_learners_classif.rpart}}, \code{\link{mlr_learners_regr.debug}}, \code{\link{mlr_learners_regr.featureless}}, -\code{\link{mlr_learners_regr.rpart}}, -\code{\link{mlr_learners}} +\code{\link{mlr_learners_regr.rpart}} } \concept{Learner} \section{Super class}{ diff --git a/man/LearnerRegr.Rd b/man/LearnerRegr.Rd index 77c07f8d5..f11cf516f 100644 --- a/man/LearnerRegr.Rd +++ b/man/LearnerRegr.Rd @@ -50,15 +50,15 @@ for established default tuning spaces. } Other Learner: -\code{\link{LearnerClassif}}, \code{\link{Learner}}, +\code{\link{LearnerClassif}}, +\code{\link{mlr_learners}}, \code{\link{mlr_learners_classif.debug}}, \code{\link{mlr_learners_classif.featureless}}, \code{\link{mlr_learners_classif.rpart}}, \code{\link{mlr_learners_regr.debug}}, \code{\link{mlr_learners_regr.featureless}}, -\code{\link{mlr_learners_regr.rpart}}, -\code{\link{mlr_learners}} +\code{\link{mlr_learners_regr.rpart}} } \concept{Learner} \section{Super class}{ diff --git a/man/Measure.Rd b/man/Measure.Rd index 171223165..775ad9ee6 100644 --- a/man/Measure.Rd +++ b/man/Measure.Rd @@ -39,14 +39,14 @@ Other Measure: \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_elapsed_time}}, \code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} } \concept{Measure} \section{Public fields}{ diff --git a/man/MeasureClassif.Rd b/man/MeasureClassif.Rd index ff5ba8a14..b862915f9 100644 --- a/man/MeasureClassif.Rd +++ b/man/MeasureClassif.Rd @@ -28,17 +28,17 @@ The default measure for classification is \code{\link[=mlr_measures_classif.ce]{ } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_elapsed_time}}, \code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} } \concept{Measure} \section{Super class}{ diff --git a/man/MeasureRegr.Rd b/man/MeasureRegr.Rd index d4a051ac8..af049a8e1 100644 --- a/man/MeasureRegr.Rd +++ b/man/MeasureRegr.Rd @@ -28,17 +28,17 @@ The default measure for regression is \code{\link[=mlr_measures_regr.mse]{regr.m } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_elapsed_time}}, \code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} } \concept{Measure} \section{Super class}{ diff --git a/man/MeasureSimilarity.Rd b/man/MeasureSimilarity.Rd index ebd9f495d..e9afb5be8 100644 --- a/man/MeasureSimilarity.Rd +++ b/man/MeasureSimilarity.Rd @@ -42,17 +42,17 @@ bmr$aggregate(msrs(c("classif.ce", "sim.jaccard"))) } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_elapsed_time}}, \code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} } \concept{Measure} \section{Super class}{ diff --git a/man/PredictionClassif.Rd b/man/PredictionClassif.Rd index aa516efc2..82a4c90f6 100644 --- a/man/PredictionClassif.Rd +++ b/man/PredictionClassif.Rd @@ -77,8 +77,8 @@ p$score(measures = msr("classif.ce")) } Other Prediction: -\code{\link{PredictionRegr}}, -\code{\link{Prediction}} +\code{\link{Prediction}}, +\code{\link{PredictionRegr}} } \concept{Prediction} \section{Super class}{ diff --git a/man/PredictionRegr.Rd b/man/PredictionRegr.Rd index f23e66d3e..ebe427cea 100644 --- a/man/PredictionRegr.Rd +++ b/man/PredictionRegr.Rd @@ -28,8 +28,8 @@ head(as.data.table(p)) } Other Prediction: -\code{\link{PredictionClassif}}, -\code{\link{Prediction}} +\code{\link{Prediction}}, +\code{\link{PredictionClassif}} } \concept{Prediction} \section{Super class}{ diff --git a/man/Resampling.Rd b/man/Resampling.Rd index d48ab4aeb..4c1c7176a 100644 --- a/man/Resampling.Rd +++ b/man/Resampling.Rd @@ -90,16 +90,16 @@ tasks. } Other Resampling: +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_bootstrap}}, -\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_custom}}, +\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_cv}}, \code{\link{mlr_resamplings_holdout}}, \code{\link{mlr_resamplings_insample}}, \code{\link{mlr_resamplings_loo}}, \code{\link{mlr_resamplings_repeated_cv}}, -\code{\link{mlr_resamplings_subsampling}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_subsampling}} } \concept{Resampling} \section{Public fields}{ diff --git a/man/ResultData.Rd b/man/ResultData.Rd index 812d04f02..62389ff23 100644 --- a/man/ResultData.Rd +++ b/man/ResultData.Rd @@ -338,6 +338,7 @@ Modified \code{self} (invisibly). \if{latex}{\out{\hypertarget{method-ResultData-marshal}{}}} \subsection{Method \code{marshal()}}{ Marshals all stored learner models. +This will do nothing to models that are already marshalled. \subsection{Usage}{ \if{html}{\out{
}}\preformatted{ResultData$marshal()}\if{html}{\out{
}} } @@ -348,6 +349,7 @@ Marshals all stored learner models. \if{latex}{\out{\hypertarget{method-ResultData-unmarshal}{}}} \subsection{Method \code{unmarshal()}}{ Unmarshals all stored learner models. +This will do nothing to models which are not marshalled. \subsection{Usage}{ \if{html}{\out{
}}\preformatted{ResultData$unmarshal()}\if{html}{\out{
}} } diff --git a/man/Task.Rd b/man/Task.Rd index da405c0c2..c0a43451c 100644 --- a/man/Task.Rd +++ b/man/Task.Rd @@ -91,6 +91,7 @@ Other Task: \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -101,8 +102,7 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} \section{Public fields}{ diff --git a/man/TaskClassif.Rd b/man/TaskClassif.Rd index 7299313d1..ee1c2bc0a 100644 --- a/man/TaskClassif.Rd +++ b/man/TaskClassif.Rd @@ -46,10 +46,11 @@ task$data(rows = 1:3, cols = task$feature_names[1:2]) } Other Task: +\code{\link{Task}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -60,8 +61,7 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} \section{Super classes}{ diff --git a/man/TaskGenerator.Rd b/man/TaskGenerator.Rd index bcc69b17f..7fe82362c 100644 --- a/man/TaskGenerator.Rd +++ b/man/TaskGenerator.Rd @@ -20,6 +20,7 @@ e.g. \code{\link[=mlr_task_generators_xor]{xor}}. } Other TaskGenerator: +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_2dnormals}}, \code{\link{mlr_task_generators_cassini}}, \code{\link{mlr_task_generators_circle}}, @@ -28,8 +29,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_simplex}}, \code{\link{mlr_task_generators_smiley}}, \code{\link{mlr_task_generators_spirals}}, -\code{\link{mlr_task_generators_xor}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_xor}} } \concept{TaskGenerator} \section{Public fields}{ diff --git a/man/TaskRegr.Rd b/man/TaskRegr.Rd index 3c85f622c..603b3ab87 100644 --- a/man/TaskRegr.Rd +++ b/man/TaskRegr.Rd @@ -36,10 +36,11 @@ task$data(rows = 1:3, cols = task$feature_names[1:2]) } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -50,8 +51,7 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} \section{Super classes}{ diff --git a/man/TaskSupervised.Rd b/man/TaskSupervised.Rd index 98f38473e..a848b0579 100644 --- a/man/TaskSupervised.Rd +++ b/man/TaskSupervised.Rd @@ -31,10 +31,11 @@ TaskSupervised$new("penguins", task_type = "classif", backend = palmerpenguins:: } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -45,8 +46,7 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} \keyword{internal} diff --git a/man/TaskUnsupervised.Rd b/man/TaskUnsupervised.Rd index 3aecbddd9..f97a110aa 100644 --- a/man/TaskUnsupervised.Rd +++ b/man/TaskUnsupervised.Rd @@ -27,10 +27,11 @@ TaskUnsupervised$new("penguins", task_type = "regr", backend = palmerpenguins::p } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -41,8 +42,7 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} \keyword{internal} diff --git a/man/as_data_backend.Rd b/man/as_data_backend.Rd index a7b45c1d4..13ad1e36c 100644 --- a/man/as_data_backend.Rd +++ b/man/as_data_backend.Rd @@ -56,8 +56,8 @@ e.g. SQL servers or \CRANpkg{duckdb}. } Other DataBackend: +\code{\link{DataBackend}}, \code{\link{DataBackendDataTable}}, -\code{\link{DataBackendMatrix}}, -\code{\link{DataBackend}} +\code{\link{DataBackendMatrix}} } \concept{DataBackend} diff --git a/man/mlr_learners.Rd b/man/mlr_learners.Rd index 3e38900ac..b06ed8839 100644 --- a/man/mlr_learners.Rd +++ b/man/mlr_learners.Rd @@ -50,9 +50,9 @@ Other Dictionary: \code{\link{mlr_tasks}} Other Learner: +\code{\link{Learner}}, \code{\link{LearnerClassif}}, \code{\link{LearnerRegr}}, -\code{\link{Learner}}, \code{\link{mlr_learners_classif.debug}}, \code{\link{mlr_learners_classif.featureless}}, \code{\link{mlr_learners_classif.rpart}}, diff --git a/man/mlr_learners_classif.debug.Rd b/man/mlr_learners_classif.debug.Rd index f5ca719ea..bef639870 100644 --- a/man/mlr_learners_classif.debug.Rd +++ b/man/mlr_learners_classif.debug.Rd @@ -102,15 +102,15 @@ for established default tuning spaces. } Other Learner: +\code{\link{Learner}}, \code{\link{LearnerClassif}}, \code{\link{LearnerRegr}}, -\code{\link{Learner}}, +\code{\link{mlr_learners}}, \code{\link{mlr_learners_classif.featureless}}, \code{\link{mlr_learners_classif.rpart}}, \code{\link{mlr_learners_regr.debug}}, \code{\link{mlr_learners_regr.featureless}}, -\code{\link{mlr_learners_regr.rpart}}, -\code{\link{mlr_learners}} +\code{\link{mlr_learners_regr.rpart}} } \concept{Learner} \section{Super classes}{ diff --git a/man/mlr_learners_classif.featureless.Rd b/man/mlr_learners_classif.featureless.Rd index 94c1b8595..2be1215af 100644 --- a/man/mlr_learners_classif.featureless.Rd +++ b/man/mlr_learners_classif.featureless.Rd @@ -68,15 +68,15 @@ for established default tuning spaces. } Other Learner: +\code{\link{Learner}}, \code{\link{LearnerClassif}}, \code{\link{LearnerRegr}}, -\code{\link{Learner}}, +\code{\link{mlr_learners}}, \code{\link{mlr_learners_classif.debug}}, \code{\link{mlr_learners_classif.rpart}}, \code{\link{mlr_learners_regr.debug}}, \code{\link{mlr_learners_regr.featureless}}, -\code{\link{mlr_learners_regr.rpart}}, -\code{\link{mlr_learners}} +\code{\link{mlr_learners_regr.rpart}} } \concept{Learner} \section{Super classes}{ diff --git a/man/mlr_learners_classif.rpart.Rd b/man/mlr_learners_classif.rpart.Rd index 139dfe41c..cdcba963f 100644 --- a/man/mlr_learners_classif.rpart.Rd +++ b/man/mlr_learners_classif.rpart.Rd @@ -82,15 +82,15 @@ for established default tuning spaces. } Other Learner: +\code{\link{Learner}}, \code{\link{LearnerClassif}}, \code{\link{LearnerRegr}}, -\code{\link{Learner}}, +\code{\link{mlr_learners}}, \code{\link{mlr_learners_classif.debug}}, \code{\link{mlr_learners_classif.featureless}}, \code{\link{mlr_learners_regr.debug}}, \code{\link{mlr_learners_regr.featureless}}, -\code{\link{mlr_learners_regr.rpart}}, -\code{\link{mlr_learners}} +\code{\link{mlr_learners_regr.rpart}} } \concept{Learner} \section{Super classes}{ diff --git a/man/mlr_learners_regr.debug.Rd b/man/mlr_learners_regr.debug.Rd index 66ffd60f2..2c2c53b23 100644 --- a/man/mlr_learners_regr.debug.Rd +++ b/man/mlr_learners_regr.debug.Rd @@ -75,15 +75,15 @@ for established default tuning spaces. } Other Learner: +\code{\link{Learner}}, \code{\link{LearnerClassif}}, \code{\link{LearnerRegr}}, -\code{\link{Learner}}, +\code{\link{mlr_learners}}, \code{\link{mlr_learners_classif.debug}}, \code{\link{mlr_learners_classif.featureless}}, \code{\link{mlr_learners_classif.rpart}}, \code{\link{mlr_learners_regr.featureless}}, -\code{\link{mlr_learners_regr.rpart}}, -\code{\link{mlr_learners}} +\code{\link{mlr_learners_regr.rpart}} } \concept{Learner} \section{Super classes}{ diff --git a/man/mlr_learners_regr.featureless.Rd b/man/mlr_learners_regr.featureless.Rd index f06fcf4d6..1f2b5d922 100644 --- a/man/mlr_learners_regr.featureless.Rd +++ b/man/mlr_learners_regr.featureless.Rd @@ -57,15 +57,15 @@ for established default tuning spaces. } Other Learner: +\code{\link{Learner}}, \code{\link{LearnerClassif}}, \code{\link{LearnerRegr}}, -\code{\link{Learner}}, +\code{\link{mlr_learners}}, \code{\link{mlr_learners_classif.debug}}, \code{\link{mlr_learners_classif.featureless}}, \code{\link{mlr_learners_classif.rpart}}, \code{\link{mlr_learners_regr.debug}}, -\code{\link{mlr_learners_regr.rpart}}, -\code{\link{mlr_learners}} +\code{\link{mlr_learners_regr.rpart}} } \concept{Learner} \section{Super classes}{ diff --git a/man/mlr_learners_regr.rpart.Rd b/man/mlr_learners_regr.rpart.Rd index 9a83fec05..eee708300 100644 --- a/man/mlr_learners_regr.rpart.Rd +++ b/man/mlr_learners_regr.rpart.Rd @@ -82,15 +82,15 @@ for established default tuning spaces. } Other Learner: +\code{\link{Learner}}, \code{\link{LearnerClassif}}, \code{\link{LearnerRegr}}, -\code{\link{Learner}}, +\code{\link{mlr_learners}}, \code{\link{mlr_learners_classif.debug}}, \code{\link{mlr_learners_classif.featureless}}, \code{\link{mlr_learners_classif.rpart}}, \code{\link{mlr_learners_regr.debug}}, -\code{\link{mlr_learners_regr.featureless}}, -\code{\link{mlr_learners}} +\code{\link{mlr_learners_regr.featureless}} } \concept{Learner} \section{Super classes}{ diff --git a/man/mlr_measures.Rd b/man/mlr_measures.Rd index 5ef8a72cb..604528e42 100644 --- a/man/mlr_measures.Rd +++ b/man/mlr_measures.Rd @@ -50,10 +50,10 @@ Other Dictionary: \code{\link{mlr_tasks}} Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, diff --git a/man/mlr_measures_aic.Rd b/man/mlr_measures_aic.Rd index c197086cb..6ebe418c0 100644 --- a/man/mlr_measures_aic.Rd +++ b/man/mlr_measures_aic.Rd @@ -55,17 +55,17 @@ msr("aic") } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_elapsed_time}}, \code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} } \concept{Measure} \section{Super class}{ diff --git a/man/mlr_measures_bic.Rd b/man/mlr_measures_bic.Rd index a5f15f1dd..9c59bb693 100644 --- a/man/mlr_measures_bic.Rd +++ b/man/mlr_measures_bic.Rd @@ -53,17 +53,17 @@ Empty ParamSet } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_elapsed_time}}, \code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} } \concept{Measure} \section{Super class}{ diff --git a/man/mlr_measures_classif.acc.Rd b/man/mlr_measures_classif.acc.Rd index 5186cf6ec..9e87c9521 100644 --- a/man/mlr_measures_classif.acc.Rd +++ b/man/mlr_measures_classif.acc.Rd @@ -61,11 +61,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -80,10 +80,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.bacc}}, diff --git a/man/mlr_measures_classif.auc.Rd b/man/mlr_measures_classif.auc.Rd index fffa0bd23..cf10b9df1 100644 --- a/man/mlr_measures_classif.auc.Rd +++ b/man/mlr_measures_classif.auc.Rd @@ -62,11 +62,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -81,21 +81,21 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.bbrier}}, \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -104,10 +104,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.bacc.Rd b/man/mlr_measures_classif.bacc.Rd index 490b7d66a..53352f385 100644 --- a/man/mlr_measures_classif.bacc.Rd +++ b/man/mlr_measures_classif.bacc.Rd @@ -71,11 +71,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -90,10 +90,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.acc}}, diff --git a/man/mlr_measures_classif.bbrier.Rd b/man/mlr_measures_classif.bbrier.Rd index 26799d900..19f3c9864 100644 --- a/man/mlr_measures_classif.bbrier.Rd +++ b/man/mlr_measures_classif.bbrier.Rd @@ -68,11 +68,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -87,21 +87,21 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -110,10 +110,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.ce.Rd b/man/mlr_measures_classif.ce.Rd index 9c7589f00..113ce681a 100644 --- a/man/mlr_measures_classif.ce.Rd +++ b/man/mlr_measures_classif.ce.Rd @@ -61,11 +61,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -80,10 +80,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.acc}}, diff --git a/man/mlr_measures_classif.costs.Rd b/man/mlr_measures_classif.costs.Rd index ef77bf921..6ba6eac81 100644 --- a/man/mlr_measures_classif.costs.Rd +++ b/man/mlr_measures_classif.costs.Rd @@ -76,17 +76,17 @@ rr$aggregate(m) } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_elapsed_time}}, \code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} Other classification measures: \code{\link{mlr_measures_classif.acc}}, @@ -97,11 +97,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -116,10 +116,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.acc}}, diff --git a/man/mlr_measures_classif.dor.Rd b/man/mlr_measures_classif.dor.Rd index c7815f55d..e96469f05 100644 --- a/man/mlr_measures_classif.dor.Rd +++ b/man/mlr_measures_classif.dor.Rd @@ -63,11 +63,11 @@ Other classification measures: \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -82,21 +82,21 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, \code{\link{mlr_measures_classif.bbrier}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -105,10 +105,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.fbeta.Rd b/man/mlr_measures_classif.fbeta.Rd index 0b8bf0d37..2fcc6f258 100644 --- a/man/mlr_measures_classif.fbeta.Rd +++ b/man/mlr_measures_classif.fbeta.Rd @@ -68,11 +68,11 @@ Other classification measures: \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -87,21 +87,21 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, \code{\link{mlr_measures_classif.bbrier}}, \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -110,10 +110,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.fdr.Rd b/man/mlr_measures_classif.fdr.Rd index fbde96ab0..5a8d6da33 100644 --- a/man/mlr_measures_classif.fdr.Rd +++ b/man/mlr_measures_classif.fdr.Rd @@ -63,11 +63,11 @@ Other classification measures: \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -82,21 +82,21 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, \code{\link{mlr_measures_classif.bbrier}}, \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -105,10 +105,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.fn.Rd b/man/mlr_measures_classif.fn.Rd index 8e5dc356f..4b6cc93c3 100644 --- a/man/mlr_measures_classif.fn.Rd +++ b/man/mlr_measures_classif.fn.Rd @@ -62,8 +62,8 @@ Other classification measures: \code{\link{mlr_measures_classif.fdr}}, \code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -78,10 +78,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -91,8 +91,8 @@ Other binary classification measures: \code{\link{mlr_measures_classif.fdr}}, \code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -101,10 +101,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.fnr.Rd b/man/mlr_measures_classif.fnr.Rd index d54334a91..7bc01c727 100644 --- a/man/mlr_measures_classif.fnr.Rd +++ b/man/mlr_measures_classif.fnr.Rd @@ -67,8 +67,8 @@ Other classification measures: \code{\link{mlr_measures_classif.fdr}}, \code{\link{mlr_measures_classif.fn}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -83,10 +83,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -96,8 +96,8 @@ Other binary classification measures: \code{\link{mlr_measures_classif.fdr}}, \code{\link{mlr_measures_classif.fn}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -106,10 +106,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.fomr.Rd b/man/mlr_measures_classif.fomr.Rd index 47a0f15b2..07414f9f8 100644 --- a/man/mlr_measures_classif.fomr.Rd +++ b/man/mlr_measures_classif.fomr.Rd @@ -64,10 +64,10 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, -\code{\link{mlr_measures_classif.fpr}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -82,10 +82,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -93,10 +93,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, -\code{\link{mlr_measures_classif.fpr}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -105,10 +105,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.fp.Rd b/man/mlr_measures_classif.fp.Rd index 0c8fca31d..93d40da41 100644 --- a/man/mlr_measures_classif.fp.Rd +++ b/man/mlr_measures_classif.fp.Rd @@ -59,8 +59,8 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, \code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, @@ -77,10 +77,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -88,8 +88,8 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, \code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, @@ -100,10 +100,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.fpr.Rd b/man/mlr_measures_classif.fpr.Rd index 581216d90..c7ba93754 100644 --- a/man/mlr_measures_classif.fpr.Rd +++ b/man/mlr_measures_classif.fpr.Rd @@ -65,8 +65,8 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, \code{\link{mlr_measures_classif.fp}}, \code{\link{mlr_measures_classif.logloss}}, @@ -83,10 +83,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -94,8 +94,8 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, \code{\link{mlr_measures_classif.fp}}, \code{\link{mlr_measures_classif.mcc}}, @@ -106,10 +106,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.logloss.Rd b/man/mlr_measures_classif.logloss.Rd index 377bcd7dd..b7eddcadf 100644 --- a/man/mlr_measures_classif.logloss.Rd +++ b/man/mlr_measures_classif.logloss.Rd @@ -63,11 +63,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, \code{\link{mlr_measures_classif.mauc_aunp}}, @@ -81,10 +81,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.acc}}, diff --git a/man/mlr_measures_classif.mauc_au1p.Rd b/man/mlr_measures_classif.mauc_au1p.Rd index 9d0ed65e6..88cc02aa4 100644 --- a/man/mlr_measures_classif.mauc_au1p.Rd +++ b/man/mlr_measures_classif.mauc_au1p.Rd @@ -80,11 +80,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1u}}, \code{\link{mlr_measures_classif.mauc_aunp}}, @@ -98,10 +98,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.acc}}, diff --git a/man/mlr_measures_classif.mauc_au1u.Rd b/man/mlr_measures_classif.mauc_au1u.Rd index 2941fda9b..ebe0cf790 100644 --- a/man/mlr_measures_classif.mauc_au1u.Rd +++ b/man/mlr_measures_classif.mauc_au1u.Rd @@ -80,11 +80,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_aunp}}, @@ -98,10 +98,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.acc}}, diff --git a/man/mlr_measures_classif.mauc_aunp.Rd b/man/mlr_measures_classif.mauc_aunp.Rd index 6f1d6da19..79c311fde 100644 --- a/man/mlr_measures_classif.mauc_aunp.Rd +++ b/man/mlr_measures_classif.mauc_aunp.Rd @@ -80,11 +80,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -98,10 +98,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.acc}}, diff --git a/man/mlr_measures_classif.mauc_aunu.Rd b/man/mlr_measures_classif.mauc_aunu.Rd index 38bec0687..26b2f622c 100644 --- a/man/mlr_measures_classif.mauc_aunu.Rd +++ b/man/mlr_measures_classif.mauc_aunu.Rd @@ -80,11 +80,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -98,10 +98,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.acc}}, diff --git a/man/mlr_measures_classif.mbrier.Rd b/man/mlr_measures_classif.mbrier.Rd index 0df60bbf7..1cdbd70ab 100644 --- a/man/mlr_measures_classif.mbrier.Rd +++ b/man/mlr_measures_classif.mbrier.Rd @@ -65,11 +65,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -83,10 +83,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other multiclass classification measures: \code{\link{mlr_measures_classif.acc}}, diff --git a/man/mlr_measures_classif.mcc.Rd b/man/mlr_measures_classif.mcc.Rd index a7e704760..91e525588 100644 --- a/man/mlr_measures_classif.mcc.Rd +++ b/man/mlr_measures_classif.mcc.Rd @@ -65,11 +65,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -83,10 +83,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -94,11 +94,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, \code{\link{mlr_measures_classif.prauc}}, @@ -106,10 +106,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.npv.Rd b/man/mlr_measures_classif.npv.Rd index 254e40aa9..8e5d7ea3a 100644 --- a/man/mlr_measures_classif.npv.Rd +++ b/man/mlr_measures_classif.npv.Rd @@ -64,11 +64,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -82,10 +82,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -93,11 +93,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.ppv}}, \code{\link{mlr_measures_classif.prauc}}, @@ -105,10 +105,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.ppv.Rd b/man/mlr_measures_classif.ppv.Rd index a2aa400d0..4c3d533dd 100644 --- a/man/mlr_measures_classif.ppv.Rd +++ b/man/mlr_measures_classif.ppv.Rd @@ -65,11 +65,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -83,10 +83,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -94,11 +94,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.prauc}}, @@ -106,10 +106,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.prauc.Rd b/man/mlr_measures_classif.prauc.Rd index 514f48d02..a8214f8ac 100644 --- a/man/mlr_measures_classif.prauc.Rd +++ b/man/mlr_measures_classif.prauc.Rd @@ -64,11 +64,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -82,10 +82,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -93,11 +93,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -105,10 +105,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.precision.Rd b/man/mlr_measures_classif.precision.Rd index a29c2433c..709700c19 100644 --- a/man/mlr_measures_classif.precision.Rd +++ b/man/mlr_measures_classif.precision.Rd @@ -65,11 +65,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -83,10 +83,10 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -94,11 +94,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -106,10 +106,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.recall.Rd b/man/mlr_measures_classif.recall.Rd index 89477ed9f..fac5af208 100644 --- a/man/mlr_measures_classif.recall.Rd +++ b/man/mlr_measures_classif.recall.Rd @@ -65,11 +65,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -83,10 +83,10 @@ Other classification measures: \code{\link{mlr_measures_classif.precision}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -94,11 +94,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -106,10 +106,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.precision}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.sensitivity.Rd b/man/mlr_measures_classif.sensitivity.Rd index 3ec613891..dac37cd5f 100644 --- a/man/mlr_measures_classif.sensitivity.Rd +++ b/man/mlr_measures_classif.sensitivity.Rd @@ -65,11 +65,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -83,10 +83,10 @@ Other classification measures: \code{\link{mlr_measures_classif.precision}}, \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -94,11 +94,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -106,10 +106,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.precision}}, \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.specificity.Rd b/man/mlr_measures_classif.specificity.Rd index 76afcf74e..907500dfb 100644 --- a/man/mlr_measures_classif.specificity.Rd +++ b/man/mlr_measures_classif.specificity.Rd @@ -65,11 +65,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -83,10 +83,10 @@ Other classification measures: \code{\link{mlr_measures_classif.precision}}, \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -94,11 +94,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -106,10 +106,10 @@ Other binary classification measures: \code{\link{mlr_measures_classif.precision}}, \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tnr}}, +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.tn.Rd b/man/mlr_measures_classif.tn.Rd index a592f0862..4684d8872 100644 --- a/man/mlr_measures_classif.tn.Rd +++ b/man/mlr_measures_classif.tn.Rd @@ -59,11 +59,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -79,8 +79,8 @@ Other classification measures: \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, \code{\link{mlr_measures_classif.tnr}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -88,11 +88,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -102,8 +102,8 @@ Other binary classification measures: \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, \code{\link{mlr_measures_classif.tnr}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.tnr.Rd b/man/mlr_measures_classif.tnr.Rd index 881c3bc08..7d005c16c 100644 --- a/man/mlr_measures_classif.tnr.Rd +++ b/man/mlr_measures_classif.tnr.Rd @@ -65,11 +65,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -85,8 +85,8 @@ Other classification measures: \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} Other binary classification measures: \code{\link{mlr_measures_classif.auc}}, @@ -94,11 +94,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -108,8 +108,8 @@ Other binary classification measures: \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, \code{\link{mlr_measures_classif.tn}}, -\code{\link{mlr_measures_classif.tpr}}, -\code{\link{mlr_measures_classif.tp}} +\code{\link{mlr_measures_classif.tp}}, +\code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} \concept{classification measures} diff --git a/man/mlr_measures_classif.tp.Rd b/man/mlr_measures_classif.tp.Rd index b1dfe4354..95c15acbf 100644 --- a/man/mlr_measures_classif.tp.Rd +++ b/man/mlr_measures_classif.tp.Rd @@ -59,11 +59,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -78,8 +78,8 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, +\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tpr}} Other binary classification measures: @@ -88,11 +88,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -101,8 +101,8 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, +\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tpr}} } \concept{binary classification measures} diff --git a/man/mlr_measures_classif.tpr.Rd b/man/mlr_measures_classif.tpr.Rd index 32bdf91c1..846077698 100644 --- a/man/mlr_measures_classif.tpr.Rd +++ b/man/mlr_measures_classif.tpr.Rd @@ -65,11 +65,11 @@ Other classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.logloss}}, \code{\link{mlr_measures_classif.mauc_au1p}}, \code{\link{mlr_measures_classif.mauc_au1u}}, @@ -84,8 +84,8 @@ Other classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, +\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tp}} Other binary classification measures: @@ -94,11 +94,11 @@ Other binary classification measures: \code{\link{mlr_measures_classif.dor}}, \code{\link{mlr_measures_classif.fbeta}}, \code{\link{mlr_measures_classif.fdr}}, -\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fn}}, +\code{\link{mlr_measures_classif.fnr}}, \code{\link{mlr_measures_classif.fomr}}, -\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.fp}}, +\code{\link{mlr_measures_classif.fpr}}, \code{\link{mlr_measures_classif.mcc}}, \code{\link{mlr_measures_classif.npv}}, \code{\link{mlr_measures_classif.ppv}}, @@ -107,8 +107,8 @@ Other binary classification measures: \code{\link{mlr_measures_classif.recall}}, \code{\link{mlr_measures_classif.sensitivity}}, \code{\link{mlr_measures_classif.specificity}}, -\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tn}}, +\code{\link{mlr_measures_classif.tnr}}, \code{\link{mlr_measures_classif.tp}} } \concept{binary classification measures} diff --git a/man/mlr_measures_debug_classif.Rd b/man/mlr_measures_debug_classif.Rd index b9bcb19f3..fd6e65e13 100644 --- a/man/mlr_measures_debug_classif.Rd +++ b/man/mlr_measures_debug_classif.Rd @@ -60,17 +60,17 @@ rr$score(measure) } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_elapsed_time}}, \code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} } \concept{Measure} \section{Super class}{ diff --git a/man/mlr_measures_elapsed_time.Rd b/man/mlr_measures_elapsed_time.Rd index e502bdec1..7647575ce 100644 --- a/man/mlr_measures_elapsed_time.Rd +++ b/man/mlr_measures_elapsed_time.Rd @@ -51,17 +51,17 @@ Empty ParamSet } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} } \concept{Measure} \section{Super class}{ diff --git a/man/mlr_measures_oob_error.Rd b/man/mlr_measures_oob_error.Rd index 284e001ef..85906bab6 100644 --- a/man/mlr_measures_oob_error.Rd +++ b/man/mlr_measures_oob_error.Rd @@ -50,17 +50,17 @@ Empty ParamSet } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_elapsed_time}}, -\code{\link{mlr_measures_selected_features}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_selected_features}} } \concept{Measure} \section{Super class}{ diff --git a/man/mlr_measures_selected_features.Rd b/man/mlr_measures_selected_features.Rd index 5a8bfacfb..2122ebb67 100644 --- a/man/mlr_measures_selected_features.Rd +++ b/man/mlr_measures_selected_features.Rd @@ -64,17 +64,17 @@ scores[, c("iteration", "selected_features")] } Other Measure: +\code{\link{Measure}}, \code{\link{MeasureClassif}}, \code{\link{MeasureRegr}}, \code{\link{MeasureSimilarity}}, -\code{\link{Measure}}, +\code{\link{mlr_measures}}, \code{\link{mlr_measures_aic}}, \code{\link{mlr_measures_bic}}, \code{\link{mlr_measures_classif.costs}}, \code{\link{mlr_measures_debug_classif}}, \code{\link{mlr_measures_elapsed_time}}, -\code{\link{mlr_measures_oob_error}}, -\code{\link{mlr_measures}} +\code{\link{mlr_measures_oob_error}} } \concept{Measure} \section{Super class}{ diff --git a/man/mlr_resamplings.Rd b/man/mlr_resamplings.Rd index 10ac604c1..a9594b1cc 100644 --- a/man/mlr_resamplings.Rd +++ b/man/mlr_resamplings.Rd @@ -47,8 +47,8 @@ Other Dictionary: Other Resampling: \code{\link{Resampling}}, \code{\link{mlr_resamplings_bootstrap}}, -\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_custom}}, +\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_cv}}, \code{\link{mlr_resamplings_holdout}}, \code{\link{mlr_resamplings_insample}}, diff --git a/man/mlr_resamplings_bootstrap.Rd b/man/mlr_resamplings_bootstrap.Rd index 3021f5e51..52e1e017c 100644 --- a/man/mlr_resamplings_bootstrap.Rd +++ b/man/mlr_resamplings_bootstrap.Rd @@ -66,15 +66,15 @@ tasks. Other Resampling: \code{\link{Resampling}}, -\code{\link{mlr_resamplings_custom_cv}}, +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_custom}}, +\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_cv}}, \code{\link{mlr_resamplings_holdout}}, \code{\link{mlr_resamplings_insample}}, \code{\link{mlr_resamplings_loo}}, \code{\link{mlr_resamplings_repeated_cv}}, -\code{\link{mlr_resamplings_subsampling}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_subsampling}} } \concept{Resampling} \section{Super class}{ diff --git a/man/mlr_resamplings_custom.Rd b/man/mlr_resamplings_custom.Rd index 4cbe316d0..bd56db451 100644 --- a/man/mlr_resamplings_custom.Rd +++ b/man/mlr_resamplings_custom.Rd @@ -43,6 +43,7 @@ tasks. Other Resampling: \code{\link{Resampling}}, +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_bootstrap}}, \code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_cv}}, @@ -50,8 +51,7 @@ Other Resampling: \code{\link{mlr_resamplings_insample}}, \code{\link{mlr_resamplings_loo}}, \code{\link{mlr_resamplings_repeated_cv}}, -\code{\link{mlr_resamplings_subsampling}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_subsampling}} } \concept{Resampling} \section{Super class}{ diff --git a/man/mlr_resamplings_custom_cv.Rd b/man/mlr_resamplings_custom_cv.Rd index a2bcbfa81..cd0a467ce 100644 --- a/man/mlr_resamplings_custom_cv.Rd +++ b/man/mlr_resamplings_custom_cv.Rd @@ -54,6 +54,7 @@ tasks. Other Resampling: \code{\link{Resampling}}, +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_bootstrap}}, \code{\link{mlr_resamplings_custom}}, \code{\link{mlr_resamplings_cv}}, @@ -61,8 +62,7 @@ Other Resampling: \code{\link{mlr_resamplings_insample}}, \code{\link{mlr_resamplings_loo}}, \code{\link{mlr_resamplings_repeated_cv}}, -\code{\link{mlr_resamplings_subsampling}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_subsampling}} } \concept{Resampling} \section{Super class}{ diff --git a/man/mlr_resamplings_cv.Rd b/man/mlr_resamplings_cv.Rd index d861dba43..5b2cd1433 100644 --- a/man/mlr_resamplings_cv.Rd +++ b/man/mlr_resamplings_cv.Rd @@ -62,15 +62,15 @@ tasks. Other Resampling: \code{\link{Resampling}}, +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_bootstrap}}, -\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_custom}}, +\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_holdout}}, \code{\link{mlr_resamplings_insample}}, \code{\link{mlr_resamplings_loo}}, \code{\link{mlr_resamplings_repeated_cv}}, -\code{\link{mlr_resamplings_subsampling}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_subsampling}} } \concept{Resampling} \section{Super class}{ diff --git a/man/mlr_resamplings_holdout.Rd b/man/mlr_resamplings_holdout.Rd index c5160d43d..09c405ead 100644 --- a/man/mlr_resamplings_holdout.Rd +++ b/man/mlr_resamplings_holdout.Rd @@ -63,15 +63,15 @@ tasks. Other Resampling: \code{\link{Resampling}}, +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_bootstrap}}, -\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_custom}}, +\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_cv}}, \code{\link{mlr_resamplings_insample}}, \code{\link{mlr_resamplings_loo}}, \code{\link{mlr_resamplings_repeated_cv}}, -\code{\link{mlr_resamplings_subsampling}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_subsampling}} } \concept{Resampling} \section{Super class}{ diff --git a/man/mlr_resamplings_insample.Rd b/man/mlr_resamplings_insample.Rd index 3a4926f8e..6592161fb 100644 --- a/man/mlr_resamplings_insample.Rd +++ b/man/mlr_resamplings_insample.Rd @@ -44,15 +44,15 @@ tasks. Other Resampling: \code{\link{Resampling}}, +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_bootstrap}}, -\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_custom}}, +\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_cv}}, \code{\link{mlr_resamplings_holdout}}, \code{\link{mlr_resamplings_loo}}, \code{\link{mlr_resamplings_repeated_cv}}, -\code{\link{mlr_resamplings_subsampling}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_subsampling}} } \concept{Resampling} \section{Super class}{ diff --git a/man/mlr_resamplings_loo.Rd b/man/mlr_resamplings_loo.Rd index cf4ad553e..f53d29283 100644 --- a/man/mlr_resamplings_loo.Rd +++ b/man/mlr_resamplings_loo.Rd @@ -66,15 +66,15 @@ tasks. Other Resampling: \code{\link{Resampling}}, +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_bootstrap}}, -\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_custom}}, +\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_cv}}, \code{\link{mlr_resamplings_holdout}}, \code{\link{mlr_resamplings_insample}}, \code{\link{mlr_resamplings_repeated_cv}}, -\code{\link{mlr_resamplings_subsampling}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_subsampling}} } \concept{Resampling} \section{Super class}{ diff --git a/man/mlr_resamplings_repeated_cv.Rd b/man/mlr_resamplings_repeated_cv.Rd index 68fbfe4e9..a674a0a91 100644 --- a/man/mlr_resamplings_repeated_cv.Rd +++ b/man/mlr_resamplings_repeated_cv.Rd @@ -73,15 +73,15 @@ tasks. Other Resampling: \code{\link{Resampling}}, +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_bootstrap}}, -\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_custom}}, +\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_cv}}, \code{\link{mlr_resamplings_holdout}}, \code{\link{mlr_resamplings_insample}}, \code{\link{mlr_resamplings_loo}}, -\code{\link{mlr_resamplings_subsampling}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_subsampling}} } \concept{Resampling} \section{Super class}{ diff --git a/man/mlr_resamplings_subsampling.Rd b/man/mlr_resamplings_subsampling.Rd index 5eb97cefe..42eedce84 100644 --- a/man/mlr_resamplings_subsampling.Rd +++ b/man/mlr_resamplings_subsampling.Rd @@ -65,15 +65,15 @@ tasks. Other Resampling: \code{\link{Resampling}}, +\code{\link{mlr_resamplings}}, \code{\link{mlr_resamplings_bootstrap}}, -\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_custom}}, +\code{\link{mlr_resamplings_custom_cv}}, \code{\link{mlr_resamplings_cv}}, \code{\link{mlr_resamplings_holdout}}, \code{\link{mlr_resamplings_insample}}, \code{\link{mlr_resamplings_loo}}, -\code{\link{mlr_resamplings_repeated_cv}}, -\code{\link{mlr_resamplings}} +\code{\link{mlr_resamplings_repeated_cv}} } \concept{Resampling} \section{Super class}{ diff --git a/man/mlr_task_generators_2dnormals.Rd b/man/mlr_task_generators_2dnormals.Rd index cc54f1fb3..9c74bde1b 100644 --- a/man/mlr_task_generators_2dnormals.Rd +++ b/man/mlr_task_generators_2dnormals.Rd @@ -45,6 +45,7 @@ str(task$data()) Other TaskGenerator: \code{\link{TaskGenerator}}, +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_cassini}}, \code{\link{mlr_task_generators_circle}}, \code{\link{mlr_task_generators_friedman1}}, @@ -52,8 +53,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_simplex}}, \code{\link{mlr_task_generators_smiley}}, \code{\link{mlr_task_generators_spirals}}, -\code{\link{mlr_task_generators_xor}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_xor}} } \concept{TaskGenerator} \section{Super class}{ diff --git a/man/mlr_task_generators_cassini.Rd b/man/mlr_task_generators_cassini.Rd index 70328e4e5..b9aa84c2d 100644 --- a/man/mlr_task_generators_cassini.Rd +++ b/man/mlr_task_generators_cassini.Rd @@ -45,6 +45,7 @@ str(task$data()) Other TaskGenerator: \code{\link{TaskGenerator}}, +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_2dnormals}}, \code{\link{mlr_task_generators_circle}}, \code{\link{mlr_task_generators_friedman1}}, @@ -52,8 +53,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_simplex}}, \code{\link{mlr_task_generators_smiley}}, \code{\link{mlr_task_generators_spirals}}, -\code{\link{mlr_task_generators_xor}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_xor}} } \concept{TaskGenerator} \section{Super class}{ diff --git a/man/mlr_task_generators_circle.Rd b/man/mlr_task_generators_circle.Rd index ae6392849..35bc4c9b6 100644 --- a/man/mlr_task_generators_circle.Rd +++ b/man/mlr_task_generators_circle.Rd @@ -44,6 +44,7 @@ str(task$data()) Other TaskGenerator: \code{\link{TaskGenerator}}, +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_2dnormals}}, \code{\link{mlr_task_generators_cassini}}, \code{\link{mlr_task_generators_friedman1}}, @@ -51,8 +52,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_simplex}}, \code{\link{mlr_task_generators_smiley}}, \code{\link{mlr_task_generators_spirals}}, -\code{\link{mlr_task_generators_xor}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_xor}} } \concept{TaskGenerator} \section{Super class}{ diff --git a/man/mlr_task_generators_friedman1.Rd b/man/mlr_task_generators_friedman1.Rd index a1527e65f..fe4ee688d 100644 --- a/man/mlr_task_generators_friedman1.Rd +++ b/man/mlr_task_generators_friedman1.Rd @@ -41,6 +41,7 @@ str(task$data()) Other TaskGenerator: \code{\link{TaskGenerator}}, +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_2dnormals}}, \code{\link{mlr_task_generators_cassini}}, \code{\link{mlr_task_generators_circle}}, @@ -48,8 +49,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_simplex}}, \code{\link{mlr_task_generators_smiley}}, \code{\link{mlr_task_generators_spirals}}, -\code{\link{mlr_task_generators_xor}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_xor}} } \concept{TaskGenerator} \section{Super class}{ diff --git a/man/mlr_task_generators_moons.Rd b/man/mlr_task_generators_moons.Rd index 691d699b2..2e7ef461f 100644 --- a/man/mlr_task_generators_moons.Rd +++ b/man/mlr_task_generators_moons.Rd @@ -44,6 +44,7 @@ str(task$data()) Other TaskGenerator: \code{\link{TaskGenerator}}, +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_2dnormals}}, \code{\link{mlr_task_generators_cassini}}, \code{\link{mlr_task_generators_circle}}, @@ -51,8 +52,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_simplex}}, \code{\link{mlr_task_generators_smiley}}, \code{\link{mlr_task_generators_spirals}}, -\code{\link{mlr_task_generators_xor}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_xor}} } \concept{TaskGenerator} \section{Super class}{ diff --git a/man/mlr_task_generators_simplex.Rd b/man/mlr_task_generators_simplex.Rd index 782b89bd6..f2e0667b3 100644 --- a/man/mlr_task_generators_simplex.Rd +++ b/man/mlr_task_generators_simplex.Rd @@ -49,6 +49,7 @@ str(task$data()) Other TaskGenerator: \code{\link{TaskGenerator}}, +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_2dnormals}}, \code{\link{mlr_task_generators_cassini}}, \code{\link{mlr_task_generators_circle}}, @@ -56,8 +57,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_moons}}, \code{\link{mlr_task_generators_smiley}}, \code{\link{mlr_task_generators_spirals}}, -\code{\link{mlr_task_generators_xor}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_xor}} } \concept{TaskGenerator} \section{Super class}{ diff --git a/man/mlr_task_generators_smiley.Rd b/man/mlr_task_generators_smiley.Rd index fc7563779..627642f4d 100644 --- a/man/mlr_task_generators_smiley.Rd +++ b/man/mlr_task_generators_smiley.Rd @@ -44,6 +44,7 @@ str(task$data()) Other TaskGenerator: \code{\link{TaskGenerator}}, +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_2dnormals}}, \code{\link{mlr_task_generators_cassini}}, \code{\link{mlr_task_generators_circle}}, @@ -51,8 +52,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_moons}}, \code{\link{mlr_task_generators_simplex}}, \code{\link{mlr_task_generators_spirals}}, -\code{\link{mlr_task_generators_xor}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_xor}} } \concept{TaskGenerator} \section{Super class}{ diff --git a/man/mlr_task_generators_spirals.Rd b/man/mlr_task_generators_spirals.Rd index 8bc7f79f0..5fe13a51e 100644 --- a/man/mlr_task_generators_spirals.Rd +++ b/man/mlr_task_generators_spirals.Rd @@ -44,6 +44,7 @@ str(task$data()) Other TaskGenerator: \code{\link{TaskGenerator}}, +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_2dnormals}}, \code{\link{mlr_task_generators_cassini}}, \code{\link{mlr_task_generators_circle}}, @@ -51,8 +52,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_moons}}, \code{\link{mlr_task_generators_simplex}}, \code{\link{mlr_task_generators_smiley}}, -\code{\link{mlr_task_generators_xor}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_xor}} } \concept{TaskGenerator} \section{Super class}{ diff --git a/man/mlr_task_generators_xor.Rd b/man/mlr_task_generators_xor.Rd index 92390d706..fd293d1bd 100644 --- a/man/mlr_task_generators_xor.Rd +++ b/man/mlr_task_generators_xor.Rd @@ -43,6 +43,7 @@ str(task$data()) Other TaskGenerator: \code{\link{TaskGenerator}}, +\code{\link{mlr_task_generators}}, \code{\link{mlr_task_generators_2dnormals}}, \code{\link{mlr_task_generators_cassini}}, \code{\link{mlr_task_generators_circle}}, @@ -50,8 +51,7 @@ Other TaskGenerator: \code{\link{mlr_task_generators_moons}}, \code{\link{mlr_task_generators_simplex}}, \code{\link{mlr_task_generators_smiley}}, -\code{\link{mlr_task_generators_spirals}}, -\code{\link{mlr_task_generators}} +\code{\link{mlr_task_generators_spirals}} } \concept{TaskGenerator} \section{Super class}{ diff --git a/man/mlr_tasks.Rd b/man/mlr_tasks.Rd index d4aea9a84..b0a203136 100644 --- a/man/mlr_tasks.Rd +++ b/man/mlr_tasks.Rd @@ -67,11 +67,11 @@ Other Dictionary: \code{\link{mlr_task_generators}} Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, diff --git a/man/mlr_tasks_boston_housing.Rd b/man/mlr_tasks_boston_housing.Rd index 77008d36b..bd1884624 100644 --- a/man/mlr_tasks_boston_housing.Rd +++ b/man/mlr_tasks_boston_housing.Rd @@ -49,11 +49,12 @@ tsk("boston_housing") } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, \code{\link{mlr_tasks_iris}}, @@ -63,7 +64,6 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_breast_cancer.Rd b/man/mlr_tasks_breast_cancer.Rd index eeed7d3e0..40315a0ce 100644 --- a/man/mlr_tasks_breast_cancer.Rd +++ b/man/mlr_tasks_breast_cancer.Rd @@ -55,11 +55,12 @@ tsk("breast_cancer") } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_german_credit}}, \code{\link{mlr_tasks_iris}}, @@ -69,7 +70,6 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_german_credit.Rd b/man/mlr_tasks_german_credit.Rd index 2cb7c987b..59f52eb92 100644 --- a/man/mlr_tasks_german_credit.Rd +++ b/man/mlr_tasks_german_credit.Rd @@ -79,11 +79,12 @@ Reports in Mathematics, Physics and Chemistry 4, Department II, Beuth University } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_iris}}, @@ -93,7 +94,6 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_iris.Rd b/man/mlr_tasks_iris.Rd index 670a43984..324a8afa1 100644 --- a/man/mlr_tasks_iris.Rd +++ b/man/mlr_tasks_iris.Rd @@ -56,11 +56,12 @@ tsk("iris") } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -70,7 +71,6 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_mtcars.Rd b/man/mlr_tasks_mtcars.Rd index 6c433ad56..a20ada95e 100644 --- a/man/mlr_tasks_mtcars.Rd +++ b/man/mlr_tasks_mtcars.Rd @@ -49,11 +49,12 @@ tsk("mtcars") } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -63,7 +64,6 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_penguins.Rd b/man/mlr_tasks_penguins.Rd index 8dad2edba..8a35b4228 100644 --- a/man/mlr_tasks_penguins.Rd +++ b/man/mlr_tasks_penguins.Rd @@ -68,11 +68,12 @@ Gorman KB, Williams TD, Fraser WR (2014). } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -82,7 +83,6 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_pima.Rd b/man/mlr_tasks_pima.Rd index 024578c61..c09365db6 100644 --- a/man/mlr_tasks_pima.Rd +++ b/man/mlr_tasks_pima.Rd @@ -49,11 +49,12 @@ tsk("pima") } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -63,7 +64,6 @@ Other Task: \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_sonar.Rd b/man/mlr_tasks_sonar.Rd index d0755329c..0d3119e0e 100644 --- a/man/mlr_tasks_sonar.Rd +++ b/man/mlr_tasks_sonar.Rd @@ -49,11 +49,12 @@ tsk("sonar") } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -63,7 +64,6 @@ Other Task: \code{\link{mlr_tasks_pima}}, \code{\link{mlr_tasks_spam}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_spam.Rd b/man/mlr_tasks_spam.Rd index 42cebc079..7ac015fc3 100644 --- a/man/mlr_tasks_spam.Rd +++ b/man/mlr_tasks_spam.Rd @@ -67,11 +67,12 @@ Dua, Dheeru, Graff, Casey (2017). } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -81,7 +82,6 @@ Other Task: \code{\link{mlr_tasks_pima}}, \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_wine.Rd b/man/mlr_tasks_wine.Rd index c17977f44..1df0f59a3 100644 --- a/man/mlr_tasks_wine.Rd +++ b/man/mlr_tasks_wine.Rd @@ -62,11 +62,12 @@ Dua, Dheeru, Graff, Casey (2017). } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -76,7 +77,6 @@ Other Task: \code{\link{mlr_tasks_pima}}, \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, -\code{\link{mlr_tasks_zoo}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_zoo}} } \concept{Task} diff --git a/man/mlr_tasks_zoo.Rd b/man/mlr_tasks_zoo.Rd index b8495f6cf..6babf04be 100644 --- a/man/mlr_tasks_zoo.Rd +++ b/man/mlr_tasks_zoo.Rd @@ -49,11 +49,12 @@ tsk("zoo") } Other Task: +\code{\link{Task}}, \code{\link{TaskClassif}}, \code{\link{TaskRegr}}, \code{\link{TaskSupervised}}, \code{\link{TaskUnsupervised}}, -\code{\link{Task}}, +\code{\link{mlr_tasks}}, \code{\link{mlr_tasks_boston_housing}}, \code{\link{mlr_tasks_breast_cancer}}, \code{\link{mlr_tasks_german_credit}}, @@ -63,7 +64,6 @@ Other Task: \code{\link{mlr_tasks_pima}}, \code{\link{mlr_tasks_sonar}}, \code{\link{mlr_tasks_spam}}, -\code{\link{mlr_tasks_wine}}, -\code{\link{mlr_tasks}} +\code{\link{mlr_tasks_wine}} } \concept{Task} diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index 1ef68c7b2..20c7a666d 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -501,7 +501,7 @@ test_that("sequential execution does not trigger marshalling", { expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 0) }) -test_that("parallel exeuction and callr marshall twice", { +test_that("parallel execution and callr marshall twice", { learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) task = tsk("iris") resampling = rsmp("holdout") diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R index 91a42d7c0..a58125244 100644 --- a/tests/testthat/test_marshal.R +++ b/tests/testthat/test_marshal.R @@ -1,4 +1,4 @@ -test_that("marshal works as expectected", { +test_that("learner methods", { learner = lrn("classif.lily") task = tsk("iris") expect_error(learner_marshal(learner), "not been trained") @@ -11,8 +11,29 @@ test_that("marshal works as expectected", { expect_true(learner_marshalled(learner)) learner$unmarshal() expect_false(learner_marshalled(learner)) +}) + +test_that("NULL", { + expect_equal(marshal_model(NULL), NULL) + expect_equal(unmarshal_model(NULL), NULL) +}) + +test_that("default method just changes class", { + x = 1 + xm = marshal_model(x) + expect_equal(class(xm), c("numeric_marshalled", "marshalled")) + expect_equal(x, unmarshal_model(xm)) +}) + +test_that("marshalling a marshalled object does nothing", { + x = 1 + xm = marshal_model(x) + expect_equal(marshal_model(xm), xm) +}) - # default does nothing - expect_equal(marshal_model(1), 1) - expect_equal(unmarshal_model(1), 1) +test_that("unmarshalling a unmarshalled object does nothing", { + x = 1 + xm = marshal_model(x) + expect_equal(unmarshal_model(xm), x) + expect_equal(unmarshal_model(unmarshal_model(xm)), x) }) diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index 1e9100cbe..deebe17a7 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -178,7 +178,7 @@ test_that("sequential execution does not trigger marshalling", { expect_equal(rr$learners[[1]]$model$marshal_count, 0) }) -test_that("parallel exeuction and callr marshall twice", { +test_that("parallel execution and callr marshall twice", { learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) task = tsk("iris") resampling = rsmp("holdout") From c9c9c6abbfb20ae01c6ca6fefce303222ed262c1 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 21 Feb 2024 09:06:35 +0100 Subject: [PATCH 16/47] Update R/Measure.R --- R/Measure.R | 1 - 1 file changed, 1 deletion(-) diff --git a/R/Measure.R b/R/Measure.R index f779e9d11..5f8eb63c7 100644 --- a/R/Measure.R +++ b/R/Measure.R @@ -173,7 +173,6 @@ Measure = R6Class("Measure", assert_measure(self, task = task, learner = learner) assert_prediction(prediction) - # FIXME: if self has property model check that not marshalled if ("requires_task" %in% self$properties && is.null(task)) { stopf("Measure '%s' requires a task", self$id) From 86e91638023cba19b1d76eeb8d60c5e126c60efa Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 21 Feb 2024 09:06:56 +0100 Subject: [PATCH 17/47] Update R/Measure.R --- R/Measure.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/Measure.R b/R/Measure.R index 5f8eb63c7..11a02fc7f 100644 --- a/R/Measure.R +++ b/R/Measure.R @@ -186,7 +186,7 @@ Measure = R6Class("Measure", stopf("Measure '%s' requires the trained model", self$id) } if ("requires_model" %in% self$properties && marshalled_model(learner$model)) { - stopf("Measure '%s' requires the trained model, but model is marshalled form", self$id) + stopf("Measure '%s' requires the trained model, but model is in marshalled form", self$id) } if ("requires_train_set" %in% self$properties && is.null(train_set)) { From e2b1b542c49453a86604b0821b2d401016f8e109 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 21 Feb 2024 09:46:41 +0100 Subject: [PATCH 18/47] docs --- R/marshal.R | 4 ++-- man/Learner.Rd | 2 +- man/LearnerClassif.Rd | 2 +- man/LearnerRegr.Rd | 2 +- man/marshalling.Rd | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/R/marshal.R b/R/marshal.R index 6dd6d8986..a49184f1a 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -4,7 +4,7 @@ #' #' @description #' Marshalling is the process of processing the model of a trained [`Learner`] so it an be successfully serialized and -#' deserialized. The naming is inspired from package [marshall](https://github.com/HenrikBengtsson/marshal) and we +#' deserialized. The naming is inspired by the [marshal package](https://github.com/HenrikBengtsson/marshal) and we #' plan to fully migrate to this package once it is on CRAN. #' The supported implementation until then should therfore be considered as a temporary solution and is likely #' to change in the future. @@ -36,7 +36,7 @@ #' For a concrete example on how to implement marshalling, see [`LearnerClassifLily`]. #' #' @param learner [`Learner`]\cr -#' The learner to marshal. +#' The learner. #' @keywords internal #' @export learner_unmarshal = function(learner) { diff --git a/man/Learner.Rd b/man/Learner.Rd index 697501794..3f2c5bb79 100644 --- a/man/Learner.Rd +++ b/man/Learner.Rd @@ -304,7 +304,7 @@ The following properties are currently standardized and understood by learners i \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"marshal"}: To save learners with this property, you need to call \verb{$marshal()} first. -If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use it's model, e.g. for prediction. +If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use its model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/LearnerClassif.Rd b/man/LearnerClassif.Rd index 478347768..c3bb02d91 100644 --- a/man/LearnerClassif.Rd +++ b/man/LearnerClassif.Rd @@ -140,7 +140,7 @@ The following properties are currently standardized and understood by learners i \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"marshal"}: To save learners with this property, you need to call \verb{$marshal()} first. -If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use it's model, e.g. for prediction. +If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use its model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/LearnerRegr.Rd b/man/LearnerRegr.Rd index f11cf516f..b28aa6b48 100644 --- a/man/LearnerRegr.Rd +++ b/man/LearnerRegr.Rd @@ -130,7 +130,7 @@ The following properties are currently standardized and understood by learners i \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"marshal"}: To save learners with this property, you need to call \verb{$marshal()} first. -If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use it's model, e.g. for prediction. +If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use its model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/marshalling.Rd b/man/marshalling.Rd index 98e642f6e..c181567bd 100644 --- a/man/marshalling.Rd +++ b/man/marshalling.Rd @@ -24,11 +24,11 @@ marshalled_model(model) } \arguments{ \item{learner}{\code{\link{Learner}}\cr -The learner to marshal.} +The learner.} } \description{ Marshalling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and -deserialized. The naming is inspired from package \href{https://github.com/HenrikBengtsson/marshal}{marshall} and we +deserialized. The naming is inspired by the \href{https://github.com/HenrikBengtsson/marshal}{marshal package} and we plan to fully migrate to this package once it is on CRAN. The supported implementation until then should therfore be considered as a temporary solution and is likely to change in the future. From c73f89235ea136ff81e962db569c72734388f9dc Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Thu, 22 Feb 2024 11:18:39 +0100 Subject: [PATCH 19/47] better approach --- NAMESPACE | 11 ++-- R/HotstartStack.R | 4 +- R/Learner.R | 6 +- R/LearnerClassifLily.R | 28 ++++----- R/Measure.R | 4 +- R/ResultData.R | 4 +- R/marshal.R | 76 +++++++++++------------- R/worker.R | 4 +- inst/testthat/helper_expectations.R | 32 +++++----- man-roxygen/param_learner_properties.R | 2 +- man/Learner.Rd | 2 +- man/LearnerClassif.Rd | 2 +- man/LearnerRegr.Rd | 2 +- man/ResultData.Rd | 4 +- man/marshaling.Rd | 64 ++++++++++++++++++++ man/marshalling.Rd | 64 -------------------- man/mlr_learners_classif.lily.Rd | 12 ++-- tests/testthat/test_HotstartStack.R | 4 +- tests/testthat/test_Learner.R | 10 ++-- tests/testthat/test_LearnerClassifLily.R | 16 ++--- tests/testthat/test_Measure.R | 2 +- tests/testthat/test_benchmark.R | 36 +++++------ tests/testthat/test_marshal.R | 22 ++++--- tests/testthat/test_resample.R | 36 +++++------ 24 files changed, 219 insertions(+), 228 deletions(-) create mode 100644 man/marshaling.Rd delete mode 100644 man/marshalling.Rd diff --git a/NAMESPACE b/NAMESPACE index 72e19638b..fdcc568d5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -87,10 +87,9 @@ S3method(fix_factor_levels,data.table) S3method(head,Task) S3method(is_missing_prediction_data,PredictionDataClassif) S3method(is_missing_prediction_data,PredictionDataRegr) -S3method(marshal_model,"NULL") S3method(marshal_model,classif_lily_model) S3method(marshal_model,default) -S3method(marshal_model,marshalled) +S3method(marshal_model,marshaled) S3method(partition,Task) S3method(partition,TaskClassif) S3method(partition,TaskRegr) @@ -108,9 +107,9 @@ S3method(set_threads,default) S3method(set_threads,list) S3method(summary,Task) S3method(tail,Task) -S3method(unmarshal_model,classif_lily_model_marshalled) +S3method(unmarshal_model,classif_lily_model_marshaled) S3method(unmarshal_model,default) -S3method(unmarshal_model,marshalled) +S3method(unmarshal_model,marshaled) export(BenchmarkResult) export(DataBackend) export(DataBackendDataTable) @@ -217,12 +216,12 @@ export(filter_prediction_data) export(install_pkgs) export(is_missing_prediction_data) export(learner_marshal) -export(learner_marshalled) +export(learner_marshaled) export(learner_unmarshal) export(lrn) export(lrns) export(marshal_model) -export(marshalled_model) +export(marshaled_model) export(mlr_learners) export(mlr_measures) export(mlr_reflections) diff --git a/R/HotstartStack.R b/R/HotstartStack.R index 431b28c9e..7d2b40f36 100644 --- a/R/HotstartStack.R +++ b/R/HotstartStack.R @@ -88,8 +88,8 @@ HotstartStack = R6Class("HotstartStack", walk(learners, function(learner) { if (is.null(learner$model)) { stopf("Learners must be trained before adding them to the hotstart stack.") - } else if (marshalled_model(learner$model)) { - stopf("Learners must be unmarshalled before adding them to the hotstart stack.") + } else if (marshaled_model(learner$model)) { + stopf("Learners must be unmarshaled before adding them to the hotstart stack.") } }) diff --git a/R/Learner.R b/R/Learner.R index 7ef30bae8..09ccba6a0 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -184,7 +184,7 @@ Learner = R6Class("Learner", #' @param ... (ignored). print = function(...) { catn(format(self), if (is.null(self$label) || is.na(self$label)) "" else paste0(": ", self$label)) - catn(str_indent("* Model:", if (is.null(self$model)) "-" else if (marshalled_model(self$model)) "" else paste0(class(self$model)[1L]))) + catn(str_indent("* Model:", if (is.null(self$model)) "-" else if (marshaled_model(self$model)) "" else paste0(class(self$model)[1L]))) catn(str_indent("* Parameters:", as_short_string(self$param_set$values, 1000L))) catn(str_indent("* Packages:", self$packages)) catn(str_indent("* Predict Types: ", replace(self$predict_types, self$predict_types == self$predict_type, paste0("[", self$predict_type, "]")))) @@ -279,8 +279,8 @@ Learner = R6Class("Learner", stopf("Cannot predict, Learner '%s' has not been trained yet", self$id) } - if (marshalled_model(self$model)) { - stopf("Cannot predict, Learner '%s' has not been unmarshalled yet", self$id) + if (marshaled_model(self$model)) { + stopf("Cannot predict, Learner '%s' has not been unmarshaled yet", self$id) } if (isTRUE(self$parallel_predict) && nbrOfWorkers() > 1L) { diff --git a/R/LearnerClassifLily.R b/R/LearnerClassifLily.R index 36440a39d..4f37e29d3 100644 --- a/R/LearnerClassifLily.R +++ b/R/LearnerClassifLily.R @@ -1,11 +1,11 @@ -#' @title Lily and Marshall +#' @title Lily and marshal #' #' @name mlr_learners_classif.lily #' @include LearnerClassifDebug.R #' #' @description -#' This learner is just like [`LearnerClassifDebug`], but can be marshalled. -#' When the `count_marshalling` parameter is `TRUE`, the model contains a `marshal_count` that will be increased +#' This learner is just like [`LearnerClassifDebug`], but can be marshaled. +#' When the `count_marshaling` parameter is `TRUE`, the model contains a `marshal_count` that will be increased #' by 1, each time `marshal_model` is called. #' #' @templateVar id classif.lily @@ -19,8 +19,8 @@ LearnerClassifLily = R6Class("LearnerClassifLily", #' Creates a new instance of this [R6][R6::R6Class] class. initialize = function() { super$initialize() - self$param_set$add(ps(count_marshalling = p_lgl(tags = c("train", "required")))) - self$param_set$values$count_marshalling = FALSE + self$param_set$add(ps(count_marshaling = p_lgl(tags = c("train", "required")))) + self$param_set$values$count_marshaling = FALSE self$properties = sort(c("marshal", self$properties)) self$man = "mlr3::mlr_learners_classif.lily" self$label = "Lily Learner" @@ -38,16 +38,16 @@ LearnerClassifLily = R6Class("LearnerClassifLily", } ), active = list( - #' @field marshalled (logical(1))\cr - #' Whether the learner has been marshalled. - marshalled = function() { - learner_marshalled(self) + #' @field marshaled (logical(1))\cr + #' Whether the learner has been marshaled. + marshaled = function() { + learner_marshaled(self) } ), private = list( .train = function(task) { model = super$.train(task) - if (self$param_set$values$count_marshalling) { + if (self$param_set$values$count_marshaling) { model$marshal_count = 0L } class(model) = "classif_lily_model" @@ -64,11 +64,11 @@ marshal_model.classif_lily_model = function(model, ...) { if (!is.null(model$marshal_count)) { model$marshal_count = model$marshal_count + 1 } - newclass = c("classif_lily_model_marshalled", "marshalled") - structure(list(model), class = newclass) + newclass = c("classif_lily_model_marshaled", "marshaled") + structure(list(marshaled = model, packages = "mlr3"), class = newclass) } #' @export -unmarshal_model.classif_lily_model_marshalled = function(model, ...) { - model[[1L]] +unmarshal_model.classif_lily_model_marshaled = function(model, ...) { + model$marshaled } diff --git a/R/Measure.R b/R/Measure.R index 11a02fc7f..14acc2eea 100644 --- a/R/Measure.R +++ b/R/Measure.R @@ -185,8 +185,8 @@ Measure = R6Class("Measure", if ("requires_model" %in% self$properties && (is.null(learner) || is.null(learner$model))) { stopf("Measure '%s' requires the trained model", self$id) } - if ("requires_model" %in% self$properties && marshalled_model(learner$model)) { - stopf("Measure '%s' requires the trained model, but model is in marshalled form", self$id) + if ("requires_model" %in% self$properties && marshaled_model(learner$model)) { + stopf("Measure '%s' requires the trained model, but model is in marshaled form", self$id) } if ("requires_train_set" %in% self$properties && is.null(train_set)) { diff --git a/R/ResultData.R b/R/ResultData.R index 1c1c18f9e..5a51acdab 100644 --- a/R/ResultData.R +++ b/R/ResultData.R @@ -244,7 +244,7 @@ ResultData = R6Class("ResultData", #' @description #' Marshals all stored learner models. - #' This will do nothing to models that are already marshalled. + #' This will do nothing to models that are already marshaled. marshal = function() { learner_state = NULL self$data$fact[, learner_state := lapply(learner_state, marshal_state_if_model)] @@ -252,7 +252,7 @@ ResultData = R6Class("ResultData", }, #' @description #' Unmarshals all stored learner models. - #' This will do nothing to models which are not marshalled. + #' This will do nothing to models which are not marshaled. unmarshal = function() { learner_state = NULL self$data$fact[, learner_state := lapply(learner_state, unmarshal_state_if_model)] diff --git a/R/marshal.R b/R/marshal.R index a49184f1a..19d7ad49c 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -1,9 +1,9 @@ #' @title (Un)marshal a Learner #' -#' @name marshalling +#' @name marshaling #' #' @description -#' Marshalling is the process of processing the model of a trained [`Learner`] so it an be successfully serialized and +#' marshaling is the process of processing the model of a trained [`Learner`] so it an be successfully serialized and #' deserialized. The naming is inspired by the [marshal package](https://github.com/HenrikBengtsson/marshal) and we #' plan to fully migrate to this package once it is on CRAN. #' The supported implementation until then should therfore be considered as a temporary solution and is likely @@ -11,29 +11,29 @@ #' #' The central functions (and the only methods that are used by `mlr3` internally) are: #' * the S3 generic `marshal_model(model, ...)`. -#' Which takes in a model and returns it in marshalled form. -#' The suffix `"_marshalled"` should be added to the class of the returned object and the root class must -#' be set to `"marshalled"`. +#' Which takes in a model and returns it in marshaled form. +#' The suffix `"_marshaled"` should be added to the class of the returned object and the root class must +#' be set to `"marshaled"`. #' * the S3 generic `unmarshal_model(model, ...)`. -#' Which takes in a model and returns it in unmarshalled form. -#' The returned object must not inherit from class `"marshalled"`. -#' * the function `marshalled_model(model)`, which returns `TRUE` if the model inherits from class `"marshalled"` +#' Which takes in a model and returns it in unmarshaled form. +#' The returned object must not inherit from class `"marshaled"`. +#' * the function `marshaled_model(model)`, which returns `TRUE` if the model inherits from class `"marshaled"` #' and `FALSE` otherwise. #' -#' In order to implement marshalling for a Learner, you only need to overload the `marshal_model` and `unmarshal_model` +#' In order to implement marshaling for a Learner, you only need to overload the `marshal_model` and `unmarshal_model` #' methods and tag the learner with the `"marshal"` property accordingly. #' -#' To make marshalling accessible in an R6-manner, you should also add the public methods `$marshal()`, `$unmarshal()` -#' and the active binding `$marshalled`. +#' To make marshaling accessible in an R6-manner, you should also add the public methods `$marshal()`, `$unmarshal()` +#' and the active binding `$marshaled`. #' To make this as convenient as possible, the functions `learner_marshal(learner)`, `learner_unmarshal(learner)` -#' and `learner_marshalled(learner)` are provided and can be called from within the public methods. +#' and `learner_marshaled(learner)` are provided and can be called from within the public methods. #' All three functions throw an error if the learner is not trained and otherwise call -#' `marshal_model()`, `unmarshal_model()` or `marshalled_model()` on the learner's model. +#' `marshal_model()`, `unmarshal_model()` or `marshaled_model()` on the learner's model. #' -#' You can verify whether you have correctly implemented marshalling by using the internal test helper -#' `expect_marshallable_learner()`. This is also run by `expect_learner()` if a task is provided. +#' You can verify whether you have correctly implemented marshaling by using the internal test helper +#' `expect_marshalable_learner()`. This is also run by `expect_learner()` if a task is provided. #' -#' For a concrete example on how to implement marshalling, see [`LearnerClassifLily`]. +#' For a concrete example on how to implement marshaling, see [`LearnerClassifLily`]. #' #' @param learner [`Learner`]\cr #' The learner. @@ -44,74 +44,68 @@ learner_unmarshal = function(learner) { if (is.null(learner$model)) { stopf("Cannot unmarshal, Learner '%s' has not been trained yet", learner$id) } - # this will do nothing if the model was not marshalled + # this will do nothing if the model was not marshaled learner$model = unmarshal_model(learner$model) invisible(learner) } -#' @rdname marshalling +#' @rdname marshaling #' @export learner_marshal = function(learner) { # no need to check for 'marshal' property as this method should only be available for such learners if (is.null(learner$model)) { stopf("Cannot marshal, Learner '%s' has not been trained yet", learner$id) } - # this will do nothing if the model was already marshalled + # this will do nothing if the model was already marshaled learner$model = marshal_model(learner$model) invisible(learner) } -#' @rdname marshalling +#' @rdname marshaling #' @export -learner_marshalled = function(learner) { +learner_marshaled = function(learner) { # no need to check for 'marshal' property as this method should only be available for such learners if (is.null(learner$model)) { - stopf("Cannot check marshalled status, Learner '%s' has not been trained yet", learner$id) + stopf("Cannot check marshaled status, Learner '%s' has not been trained yet", learner$id) } - marshalled_model(learner$model) + marshaled_model(learner$model) } -#' @rdname marshalling +#' @rdname marshaling #' @export marshal_model = function(model, ...) { UseMethod("marshal_model") } -#' @rdname marshalling +#' @rdname marshaling #' @export unmarshal_model = function(model, ...) { + if (marshaled_model(model) && is.character(model$packages)) { + require_namespaces(model$packages) + } UseMethod("unmarshal_model") } -#' @rdname marshalling +#' @rdname marshaling #' @export -marshalled_model = function(model) { - test_class(model, "marshalled") +marshaled_model = function(model) { + test_class(model, "marshaled") } #' @export marshal_model.default = function(model, ...) { classes = class(model) - class(model) = c(paste0(classes, "_marshalled"), "marshalled") - model + structure(list(marshaled = model), class = c(paste0(classes, "_marshaled"), "marshaled")) } #' @export -marshal_model.marshalled = function(model, ...) { +marshal_model.marshaled = function(model, ...) { model } #' @export -marshal_model.NULL = function(model, ...) { - # NULL cannot have a class, so it is not covered by the default method - model -} - -#' @export -unmarshal_model.marshalled = function(model, ...) { - classes = class(model)[-length(class(model))] - class(model) = gsub("_marshalled$", "", classes) - model +unmarshal_model.marshaled = function(model, ...) { + model[["marshaled"]] } #' @export diff --git a/R/worker.R b/R/worker.R index 0de22d6d1..33af61fb2 100644 --- a/R/worker.R +++ b/R/worker.R @@ -72,7 +72,7 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL log = append_log(NULL, "train", result$log$class, result$log$msg) train_time = result$elapsed - # unmarshal_model does nothing if the model was not marshalled + # unmarshal_model does nothing if the model was not marshaled # We always want to unmarshal the model, because either: @@ -280,7 +280,7 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, lg$debug("Erasing stored model for learner '%s'", learner$id) learner$state$model = NULL } else if ("marshal" %in% learner$properties && !is_sequential) { - lg$debug("Marshalling model for learner '%s'", learner$id) + lg$debug("marshaling model for learner '%s'", learner$id) learner$model = marshal_model(learner$model) } diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index a499faf36..2403e34df 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -381,7 +381,7 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { testthat::expect_identical(lrn$task_type, task$task_type) if ("marshal" %in% lrn$properties) { - expect_marshallable_learner(lrn, task) + expect_marshalable_learner(lrn, task) } } else if ("marshal" %in% lrn$properties) { message("Cannot test 'marshal' property of the learner as no task is provided.") @@ -393,7 +393,7 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { } -expect_marshallable_learner = function(learner, task) { +expect_marshalable_learner = function(learner, task) { expect_true("marshal" %in% learner$properties) learner$state = NULL @@ -403,39 +403,39 @@ expect_marshallable_learner = function(learner, task) { expect_true(has_public(learner, "marshal") && test_function(learner$marshal, nargs = 0)) expect_true(has_public(learner, "unmarshal") && test_function(learner$unmarshal, nargs = 0)) - expect_true(has_public(learner, "marshalled")) + expect_true(has_public(learner, "marshaled")) # (un)marshal only possible after training expect_error(learner$marshal(), "has not been trained") expect_error(learner$unmarshal(), "has not been trained") - expect_error(learner$marshalled, "has not been trained") + expect_error(learner$marshaled, "has not been trained") learner$train(task) model = learner$model class_prev = class(model) - expect_false(learner$marshalled) - expect_equal(marshalled_model(learner$model), learner$marshalled) + expect_false(learner$marshaled) + expect_equal(marshaled_model(learner$model), learner$marshaled) expect_invisible(learner$marshal()) - expect_true(learner$marshalled) - expect_equal(marshalled_model(learner$model), learner$marshalled) + expect_true(learner$marshaled) + expect_equal(marshaled_model(learner$model), learner$marshaled) - # cannot predict with marshalled learner - expect_error(learner$predict(task), "has not been unmarshalled") + # cannot predict with marshaled learner + expect_error(learner$predict(task), "has not been unmarshaled") - # unmarshalling works + # unmarshaling works expect_invisible(learner$unmarshal()) - # can predict after unmarshalling + # can predict after unmarshaling expect_prediction(learner$predict(task)) # model is reset expect_equal(learner$model, model) - # marshalled is set accordingly - expect_false(learner$marshalled) + # marshaled is set accordingly + expect_false(learner$marshaled) expect_equal(class(learner$model), class_prev) - # when re-training, marshalled is reset + # when re-training, marshaled is reset learner$predict(task) - expect_false(learner$train(task)$marshalled) + expect_false(learner$train(task)$marshaled) diff --git a/man-roxygen/param_learner_properties.R b/man-roxygen/param_learner_properties.R index df51e66a8..b46ddf54d 100644 --- a/man-roxygen/param_learner_properties.R +++ b/man-roxygen/param_learner_properties.R @@ -8,4 +8,4 @@ #' * `"selected_features"`: The learner supports extraction of the set of selected features, i.e. comes with a `$selected_features()` extractor function (see section on optional extractors in [Learner]). #' * `"oob_error"`: The learner supports extraction of estimated out of bag error, i.e. comes with a `oob_error()` extractor function (see section on optional extractors in [Learner]). #' * `"marshal"`: To save learners with this property, you need to call `$marshal()` first. -#' If a learner is in a marshalled state, you call first need to call `$unmarshal()` to use its model, e.g. for prediction. +#' If a learner is in a marshaled state, you call first need to call `$unmarshal()` to use its model, e.g. for prediction. diff --git a/man/Learner.Rd b/man/Learner.Rd index 3f2c5bb79..30b53a03e 100644 --- a/man/Learner.Rd +++ b/man/Learner.Rd @@ -304,7 +304,7 @@ The following properties are currently standardized and understood by learners i \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"marshal"}: To save learners with this property, you need to call \verb{$marshal()} first. -If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use its model, e.g. for prediction. +If a learner is in a marshaled state, you call first need to call \verb{$unmarshal()} to use its model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/LearnerClassif.Rd b/man/LearnerClassif.Rd index c3bb02d91..4fe514089 100644 --- a/man/LearnerClassif.Rd +++ b/man/LearnerClassif.Rd @@ -140,7 +140,7 @@ The following properties are currently standardized and understood by learners i \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"marshal"}: To save learners with this property, you need to call \verb{$marshal()} first. -If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use its model, e.g. for prediction. +If a learner is in a marshaled state, you call first need to call \verb{$unmarshal()} to use its model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/LearnerRegr.Rd b/man/LearnerRegr.Rd index b28aa6b48..4c695fc6e 100644 --- a/man/LearnerRegr.Rd +++ b/man/LearnerRegr.Rd @@ -130,7 +130,7 @@ The following properties are currently standardized and understood by learners i \item \code{"selected_features"}: The learner supports extraction of the set of selected features, i.e. comes with a \verb{$selected_features()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"oob_error"}: The learner supports extraction of estimated out of bag error, i.e. comes with a \code{oob_error()} extractor function (see section on optional extractors in \link{Learner}). \item \code{"marshal"}: To save learners with this property, you need to call \verb{$marshal()} first. -If a learner is in a marshalled state, you call first need to call \verb{$unmarshal()} to use its model, e.g. for prediction. +If a learner is in a marshaled state, you call first need to call \verb{$unmarshal()} to use its model, e.g. for prediction. }} \item{\code{data_formats}}{(\code{character()})\cr diff --git a/man/ResultData.Rd b/man/ResultData.Rd index 62389ff23..d276815cf 100644 --- a/man/ResultData.Rd +++ b/man/ResultData.Rd @@ -338,7 +338,7 @@ Modified \code{self} (invisibly). \if{latex}{\out{\hypertarget{method-ResultData-marshal}{}}} \subsection{Method \code{marshal()}}{ Marshals all stored learner models. -This will do nothing to models that are already marshalled. +This will do nothing to models that are already marshaled. \subsection{Usage}{ \if{html}{\out{
}}\preformatted{ResultData$marshal()}\if{html}{\out{
}} } @@ -349,7 +349,7 @@ This will do nothing to models that are already marshalled. \if{latex}{\out{\hypertarget{method-ResultData-unmarshal}{}}} \subsection{Method \code{unmarshal()}}{ Unmarshals all stored learner models. -This will do nothing to models which are not marshalled. +This will do nothing to models which are not marshaled. \subsection{Usage}{ \if{html}{\out{
}}\preformatted{ResultData$unmarshal()}\if{html}{\out{
}} } diff --git a/man/marshaling.Rd b/man/marshaling.Rd new file mode 100644 index 000000000..4647c3fdd --- /dev/null +++ b/man/marshaling.Rd @@ -0,0 +1,64 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/marshal.R +\name{marshaling} +\alias{marshaling} +\alias{learner_unmarshal} +\alias{learner_marshal} +\alias{learner_marshaled} +\alias{marshal_model} +\alias{unmarshal_model} +\alias{marshaled_model} +\title{(Un)marshal a Learner} +\usage{ +learner_unmarshal(learner) + +learner_marshal(learner) + +learner_marshaled(learner) + +marshal_model(model, ...) + +unmarshal_model(model, ...) + +marshaled_model(model) +} +\arguments{ +\item{learner}{\code{\link{Learner}}\cr +The learner.} +} +\description{ +marshaling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and +deserialized. The naming is inspired by the \href{https://github.com/HenrikBengtsson/marshal}{marshal package} and we +plan to fully migrate to this package once it is on CRAN. +The supported implementation until then should therfore be considered as a temporary solution and is likely +to change in the future. + +The central functions (and the only methods that are used by \code{mlr3} internally) are: +\itemize{ +\item the S3 generic \code{marshal_model(model, ...)}. +Which takes in a model and returns it in marshaled form. +The suffix \code{"_marshaled"} should be added to the class of the returned object and the root class must +be set to \code{"marshaled"}. +\item the S3 generic \code{unmarshal_model(model, ...)}. +Which takes in a model and returns it in unmarshaled form. +The returned object must not inherit from class \code{"marshaled"}. +\item the function \code{marshaled_model(model)}, which returns \code{TRUE} if the model inherits from class \code{"marshaled"} +and \code{FALSE} otherwise. +} + +In order to implement marshaling for a Learner, you only need to overload the \code{marshal_model} and \code{unmarshal_model} +methods and tag the learner with the \code{"marshal"} property accordingly. + +To make marshaling accessible in an R6-manner, you should also add the public methods \verb{$marshal()}, \verb{$unmarshal()} +and the active binding \verb{$marshaled}. +To make this as convenient as possible, the functions \code{learner_marshal(learner)}, \code{learner_unmarshal(learner)} +and \code{learner_marshaled(learner)} are provided and can be called from within the public methods. +All three functions throw an error if the learner is not trained and otherwise call +\code{marshal_model()}, \code{unmarshal_model()} or \code{marshaled_model()} on the learner's model. + +You can verify whether you have correctly implemented marshaling by using the internal test helper +\code{expect_marshalable_learner()}. This is also run by \code{expect_learner()} if a task is provided. + +For a concrete example on how to implement marshaling, see \code{\link{LearnerClassifLily}}. +} +\keyword{internal} diff --git a/man/marshalling.Rd b/man/marshalling.Rd deleted file mode 100644 index c181567bd..000000000 --- a/man/marshalling.Rd +++ /dev/null @@ -1,64 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/marshal.R -\name{marshalling} -\alias{marshalling} -\alias{learner_unmarshal} -\alias{learner_marshal} -\alias{learner_marshalled} -\alias{marshal_model} -\alias{unmarshal_model} -\alias{marshalled_model} -\title{(Un)marshal a Learner} -\usage{ -learner_unmarshal(learner) - -learner_marshal(learner) - -learner_marshalled(learner) - -marshal_model(model, ...) - -unmarshal_model(model, ...) - -marshalled_model(model) -} -\arguments{ -\item{learner}{\code{\link{Learner}}\cr -The learner.} -} -\description{ -Marshalling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and -deserialized. The naming is inspired by the \href{https://github.com/HenrikBengtsson/marshal}{marshal package} and we -plan to fully migrate to this package once it is on CRAN. -The supported implementation until then should therfore be considered as a temporary solution and is likely -to change in the future. - -The central functions (and the only methods that are used by \code{mlr3} internally) are: -\itemize{ -\item the S3 generic \code{marshal_model(model, ...)}. -Which takes in a model and returns it in marshalled form. -The suffix \code{"_marshalled"} should be added to the class of the returned object and the root class must -be set to \code{"marshalled"}. -\item the S3 generic \code{unmarshal_model(model, ...)}. -Which takes in a model and returns it in unmarshalled form. -The returned object must not inherit from class \code{"marshalled"}. -\item the function \code{marshalled_model(model)}, which returns \code{TRUE} if the model inherits from class \code{"marshalled"} -and \code{FALSE} otherwise. -} - -In order to implement marshalling for a Learner, you only need to overload the \code{marshal_model} and \code{unmarshal_model} -methods and tag the learner with the \code{"marshal"} property accordingly. - -To make marshalling accessible in an R6-manner, you should also add the public methods \verb{$marshal()}, \verb{$unmarshal()} -and the active binding \verb{$marshalled}. -To make this as convenient as possible, the functions \code{learner_marshal(learner)}, \code{learner_unmarshal(learner)} -and \code{learner_marshalled(learner)} are provided and can be called from within the public methods. -All three functions throw an error if the learner is not trained and otherwise call -\code{marshal_model()}, \code{unmarshal_model()} or \code{marshalled_model()} on the learner's model. - -You can verify whether you have correctly implemented marshalling by using the internal test helper -\code{expect_marshallable_learner()}. This is also run by \code{expect_learner()} if a task is provided. - -For a concrete example on how to implement marshalling, see \code{\link{LearnerClassifLily}}. -} -\keyword{internal} diff --git a/man/mlr_learners_classif.lily.Rd b/man/mlr_learners_classif.lily.Rd index 376c308b9..5125b14d3 100644 --- a/man/mlr_learners_classif.lily.Rd +++ b/man/mlr_learners_classif.lily.Rd @@ -3,10 +3,10 @@ \name{mlr_learners_classif.lily} \alias{mlr_learners_classif.lily} \alias{LearnerClassifLily} -\title{Lily and Marshall} +\title{Lily and marshal} \description{ -This learner is just like \code{\link{LearnerClassifDebug}}, but can be marshalled. -When the \code{count_marshalling} parameter is \code{TRUE}, the model contains a \code{marshal_count} that will be increased +This learner is just like \code{\link{LearnerClassifDebug}}, but can be marshaled. +When the \code{count_marshaling} parameter is \code{TRUE}, the model contains a \code{marshal_count} that will be increased by 1, each time \code{marshal_model} is called. } \section{Dictionary}{ @@ -47,7 +47,7 @@ lrn("classif.lily") warning_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr x \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr iter \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr - count_marshalling \tab logical \tab - \tab TRUE, FALSE \tab - \cr + count_marshaling \tab logical \tab - \tab TRUE, FALSE \tab - \cr } } @@ -57,8 +57,8 @@ lrn("classif.lily") \section{Active bindings}{ \if{html}{\out{
}} \describe{ -\item{\code{marshalled}}{(logical(1))\cr -Whether the learner has been marshalled.} +\item{\code{marshaled}}{(logical(1))\cr +Whether the learner has been marshaled.} } \if{html}{\out{
}} } diff --git a/tests/testthat/test_HotstartStack.R b/tests/testthat/test_HotstartStack.R index a074f5e9d..26408e4fc 100644 --- a/tests/testthat/test_HotstartStack.R +++ b/tests/testthat/test_HotstartStack.R @@ -410,10 +410,10 @@ test_that("HotstartStack threshold works", { expect_data_table(hot$stack, nrows = 1) }) -test_that("error when adding marshalled learner", { +test_that("error when adding marshaled learner", { hot = HotstartStack$new() learner = lrn("classif.lily") learner$train(tsk("iris")) learner$marshal() - expect_error(hot$add(learner), "unmarshalled") + expect_error(hot$add(learner), "unmarshaled") }) diff --git a/tests/testthat/test_Learner.R b/tests/testthat/test_Learner.R index 769a6c86f..41e7050a6 100644 --- a/tests/testthat/test_Learner.R +++ b/tests/testthat/test_Learner.R @@ -325,18 +325,18 @@ test_that("Models can be replaced", { expect_equal(learner$model$location, 1) }) -test_that("marshalling and encapsulation", { +test_that("marshaling and encapsulation", { task = tsk("iris") - learner = lrn("classif.lily", count_marshalling = TRUE) + learner = lrn("classif.lily", count_marshaling = TRUE) - # callr encapsulation causes marshalling + # callr encapsulation causes marshaling learner$encapsulate = c(train = "callr") learner$train(task) expect_equal(learner$model$marshal_count, 1) - expect_false(learner$marshalled) + expect_false(learner$marshaled) expect_prediction(learner$predict(task)) - # no marshalling with no other encapsulation + # no marshaling with no other encapsulation learner$encapsulate = c(train = "none") learner$train(task) expect_equal(learner$model$marshal_count, 0) diff --git a/tests/testthat/test_LearnerClassifLily.R b/tests/testthat/test_LearnerClassifLily.R index 57691b090..9d35ef495 100644 --- a/tests/testthat/test_LearnerClassifLily.R +++ b/tests/testthat/test_LearnerClassifLily.R @@ -2,19 +2,19 @@ test_that("lily", { task = tsk("iris") learner = lrn("classif.lily") learner$train(task) - expect_false(learner$marshalled) + expect_false(learner$marshaled) learner$marshal() - expect_true(learner$marshalled) - expect_error(learner$predict(task), "has not been unmarshalled") + expect_true(learner$marshaled) + expect_error(learner$predict(task), "has not been unmarshaled") learner$unmarshal() expect_learner(learner, task) }) test_that("marshal count works", { - # to mock that marshalling behaves as expected, we need to be know how often it happened + # to mock that marshaling behaves as expected, we need to be know how often it happened # note that this means that marshal_model modifies the model in a permanent way, i.e. this is not reversed by # unmarshal_model. - learner = lrn("classif.lily", count_marshalling = TRUE) + learner = lrn("classif.lily", count_marshaling = TRUE) task = tsk("iris") learner$train(task) expect_equal(learner$model$marshal_count, 0) @@ -23,9 +23,9 @@ test_that("marshal count works", { learner$marshal()$unmarshal() expect_equal(learner$model$marshal_count, 2) - # TO make the lily learner more realistic (i.e. (un)marshalling leaves the object unchanged) - # the count_marshalling parameter can also be set to FALSE - learner2 = lrn("classif.lily", count_marshalling = FALSE) + # TO make the lily learner more realistic (i.e. (un)marshaling leaves the object unchanged) + # the count_marshaling parameter can also be set to FALSE + learner2 = lrn("classif.lily", count_marshaling = FALSE) learner2$train(task) expect_true(is.null(learner2$model$marshal_count)) model1 = learner2$model diff --git a/tests/testthat/test_Measure.R b/tests/testthat/test_Measure.R index fcae25fa2..5acf1e6eb 100644 --- a/tests/testthat/test_Measure.R +++ b/tests/testthat/test_Measure.R @@ -131,7 +131,7 @@ test_that("time_train is > 0", { expect_gte(res$time_train, 0) }) -test_that("scoring fails when measure requires_model, but model is in marshalled state", { +test_that("scoring fails when measure requires_model, but model is in marshaled state", { measure = msr("classif.acc") measure$properties = c(measure$properties, "requires_model") diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index 20c7a666d..2fbda9a0f 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -478,8 +478,8 @@ test_that("param_values in benchmark", { }) -test_that("parallel execution automatically triggers marshalling", { - learner = lrn("classif.lily", count_marshalling = TRUE) +test_that("parallel execution automatically triggers marshaling", { + learner = lrn("classif.lily", count_marshaling = TRUE) task = tsk("iris") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) @@ -487,11 +487,11 @@ test_that("parallel execution automatically triggers marshalling", { benchmark(design, store_models = TRUE, unmarshal = TRUE) }) expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 1) - expect_false(bmr$resample_result(1)$learners[[1]]$marshalled) + expect_false(bmr$resample_result(1)$learners[[1]]$marshaled) }) -test_that("sequential execution does not trigger marshalling", { - learner = lrn("classif.lily", count_marshalling = TRUE) +test_that("sequential execution does not trigger marshaling", { + learner = lrn("classif.lily", count_marshaling = TRUE) task = tsk("iris") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) @@ -501,8 +501,8 @@ test_that("sequential execution does not trigger marshalling", { expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 0) }) -test_that("parallel execution and callr marshall twice", { - learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) +test_that("parallel execution and callr marshal twice", { + learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) task = tsk("iris") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) @@ -510,34 +510,34 @@ test_that("parallel execution and callr marshall twice", { benchmark(design, store_models = TRUE, unmarshal = TRUE) }) expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 2) - expect_false(bmr$resample_result(1)$learners[[1]]$marshalled) + expect_false(bmr$resample_result(1)$learners[[1]]$marshaled) }) test_that("unmarshal parameter is respected", { - learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) + learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) task = tsk("iris") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) bmr = with_future(future::multisession, { list( - marshalled = benchmark(design, store_models = TRUE, unmarshal = FALSE), - unmarshalled = benchmark(design, store_models = TRUE, unmarshal = TRUE) + marshaled = benchmark(design, store_models = TRUE, unmarshal = FALSE), + unmarshaled = benchmark(design, store_models = TRUE, unmarshal = TRUE) ) }) - expect_false(bmr$unmarshalled$resample_result(1)$learners[[1]]$marshalled) - expect_true(bmr$marshalled$resample_result(1)$learners[[1]]$marshalled) + expect_false(bmr$unmarshaled$resample_result(1)$learners[[1]]$marshaled) + expect_true(bmr$marshaled$resample_result(1)$learners[[1]]$marshaled) }) -test_that("BenchmarkResult can be (un)marshalled", { +test_that("BenchmarkResult can be (un)marshaled", { bmr = benchmark(benchmark_grid(tsk("iris"), lrn("classif.lily"), rsmp("holdout")), store_models = TRUE) - expect_false(bmr$resample_result(1)$learners[[1]]$marshalled) + expect_false(bmr$resample_result(1)$learners[[1]]$marshaled) bmr$marshal() - expect_true(bmr$resample_result(1)$learners[[1]]$marshalled) + expect_true(bmr$resample_result(1)$learners[[1]]$marshaled) bmr$unmarshal() - expect_false(bmr$resample_result(1)$learners[[1]]$marshalled) + expect_false(bmr$resample_result(1)$learners[[1]]$marshaled) - # also works with non-marshallable learner + # also works with non-marshalable learner bmr1 = benchmark(benchmark_grid(tsk("iris"), lrn("classif.featureless"), rsmp("holdout")), store_models = TRUE) model = bmr1$resample_result(1)$learners[[1]]$model bmr1$unmarshal() diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R index a58125244..1b5d4633f 100644 --- a/tests/testthat/test_marshal.R +++ b/tests/testthat/test_marshal.R @@ -3,37 +3,35 @@ test_that("learner methods", { task = tsk("iris") expect_error(learner_marshal(learner), "not been trained") expect_error(learner_unmarshal(learner), "not been trained") - expect_error(learner_marshalled(learner), "not been trained") + expect_error(learner_marshaled(learner), "not been trained") learner$train(task) - expect_false(learner_marshalled(learner)) + expect_false(learner_marshaled(learner)) learner$marshal() - expect_true(learner_marshalled(learner)) + expect_true(learner_marshaled(learner)) learner$unmarshal() - expect_false(learner_marshalled(learner)) -}) - -test_that("NULL", { - expect_equal(marshal_model(NULL), NULL) - expect_equal(unmarshal_model(NULL), NULL) + expect_false(learner_marshaled(learner)) }) test_that("default method just changes class", { x = 1 xm = marshal_model(x) - expect_equal(class(xm), c("numeric_marshalled", "marshalled")) + expect_equal(class(xm), c("numeric_marshaled", "marshaled")) + expect_equal(xm[["marshaled"]], x) expect_equal(x, unmarshal_model(xm)) }) -test_that("marshalling a marshalled object does nothing", { +test_that("marshaling a marshaled object does nothing", { x = 1 xm = marshal_model(x) expect_equal(marshal_model(xm), xm) }) -test_that("unmarshalling a unmarshalled object does nothing", { +test_that("unmarshaling a unmarshaled object does nothing", { x = 1 xm = marshal_model(x) expect_equal(unmarshal_model(xm), x) expect_equal(unmarshal_model(unmarshal_model(xm)), x) }) + +test_that("versions are required") diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index deebe17a7..6dd046614 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -157,19 +157,19 @@ test_that("as_resample_result works for result data", { expect_class(rr2, "ResampleResult") }) -test_that("parallel execution automatically triggers marshalling", { - learner = lrn("classif.lily", count_marshalling = TRUE) +test_that("parallel execution automatically triggers marshaling", { + learner = lrn("classif.lily", count_marshaling = TRUE) task = tsk("iris") resampling = rsmp("holdout") rr = with_future(future::multisession, { resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) }) expect_equal(rr$learners[[1]]$model$marshal_count, 1) - expect_false(rr$learners[[1]]$marshalled) + expect_false(rr$learners[[1]]$marshaled) }) -test_that("sequential execution does not trigger marshalling", { - learner = lrn("classif.lily", count_marshalling = TRUE) +test_that("sequential execution does not trigger marshaling", { + learner = lrn("classif.lily", count_marshaling = TRUE) task = tsk("iris") resampling = rsmp("holdout") rr = with_future(future::sequential, { @@ -178,41 +178,41 @@ test_that("sequential execution does not trigger marshalling", { expect_equal(rr$learners[[1]]$model$marshal_count, 0) }) -test_that("parallel execution and callr marshall twice", { - learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) +test_that("parallel execution and callr marshal twice", { + learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) task = tsk("iris") resampling = rsmp("holdout") rr = with_future(future::multisession, { resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) }) expect_equal(rr$learners[[1]]$model$marshal_count, 2) - expect_false(rr$learners[[1]]$marshalled) + expect_false(rr$learners[[1]]$marshaled) }) test_that("unmarshal parameter is respected", { - learner = lrn("classif.lily", count_marshalling = TRUE, encapsulate = c(train = "callr")) + learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) task = tsk("iris") resampling = rsmp("holdout") rr = with_future(future::multisession, { list( - marshalled = resample(task, learner, resampling, store_models = TRUE, unmarshal = FALSE), - unmarshalled = resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) + marshaled = resample(task, learner, resampling, store_models = TRUE, unmarshal = FALSE), + unmarshaled = resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) ) }) - expect_false(rr$unmarshalled$learners[[1]]$marshalled) - expect_true(rr$marshalled$learners[[1]]$marshalled) + expect_false(rr$unmarshaled$learners[[1]]$marshaled) + expect_true(rr$marshaled$learners[[1]]$marshaled) }) -test_that("ResampleResult can be (un)marshalled", { +test_that("ResampleResult can be (un)marshaled", { rr = resample(tsk("iris"), lrn("classif.lily"), rsmp("holdout"), store_models = TRUE) - expect_false(rr$learners[[1]]$marshalled) + expect_false(rr$learners[[1]]$marshaled) rr$marshal() - expect_true(rr$learners[[1]]$marshalled) + expect_true(rr$learners[[1]]$marshaled) rr$unmarshal() - expect_false(rr$learners[[1]]$marshalled) + expect_false(rr$learners[[1]]$marshaled) - # also works with non-marshallable learner + # also works with non-marshalable learner rr1 = resample(tsk("iris"), lrn("classif.featureless"), rsmp("holdout"), store_models = TRUE) model = rr1$learners[[1]]$model rr1$unmarshal() From e0c53eac3c38f347b1cb1ab78450ae6cbe714944 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Thu, 22 Feb 2024 11:21:55 +0100 Subject: [PATCH 20/47] docs --- R/marshal.R | 9 ++++++--- man/marshaling.Rd | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/R/marshal.R b/R/marshal.R index 19d7ad49c..3bc2793a4 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -12,10 +12,13 @@ #' The central functions (and the only methods that are used by `mlr3` internally) are: #' * the S3 generic `marshal_model(model, ...)`. #' Which takes in a model and returns it in marshaled form. -#' The suffix `"_marshaled"` should be added to the class of the returned object and the root class must -#' be set to `"marshaled"`. +#' The marshaled object should be a list with named elements `marshaled` and `packages`, where the former contains +#' the actual marshaled object, and the latter the packages required to unmarshal it. +#' This list should have as classes the classes of the original object with the suffix `"_marshaled"` added and the +#' root class should be set to `"marshaled"`. #' * the S3 generic `unmarshal_model(model, ...)`. -#' Which takes in a model and returns it in unmarshaled form. +#' Which takes in the marshaled model and returns it in unmarshaled form. +#' The generic takes care that the packages specified during `"marshal"` are loaded, and errs if they are not. #' The returned object must not inherit from class `"marshaled"`. #' * the function `marshaled_model(model)`, which returns `TRUE` if the model inherits from class `"marshaled"` #' and `FALSE` otherwise. diff --git a/man/marshaling.Rd b/man/marshaling.Rd index 4647c3fdd..fb4b6b50a 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -37,10 +37,13 @@ The central functions (and the only methods that are used by \code{mlr3} interna \itemize{ \item the S3 generic \code{marshal_model(model, ...)}. Which takes in a model and returns it in marshaled form. -The suffix \code{"_marshaled"} should be added to the class of the returned object and the root class must -be set to \code{"marshaled"}. +The marshaled object should be a list with named elements \code{marshaled} and \code{packages}, where the former contains +the actual marshaled object, and the latter the packages required to unmarshal it. +This list should have as classes the classes of the original object with the suffix \code{"_marshaled"} added and the +root class should be set to \code{"marshaled"}. \item the S3 generic \code{unmarshal_model(model, ...)}. -Which takes in a model and returns it in unmarshaled form. +Which takes in the marshaled model and returns it in unmarshaled form. +The generic takes care that the packages specified during \code{"marshal"} are loaded, and errs if they are not. The returned object must not inherit from class \code{"marshaled"}. \item the function \code{marshaled_model(model)}, which returns \code{TRUE} if the model inherits from class \code{"marshaled"} and \code{FALSE} otherwise. From ffffc3054518d0ff0f39ebf66ac15af0ec42aba5 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Thu, 22 Feb 2024 18:10:14 +0100 Subject: [PATCH 21/47] add clone argument and optimize worker --- NAMESPACE | 2 -- R/Learner.R | 2 ++ R/marshal.R | 4 ++-- R/worker.R | 19 ++++++++----------- man/marshaling.Rd | 4 ++-- tests/testthat/test_benchmark.R | 4 ++-- tests/testthat/test_marshal.R | 2 -- tests/testthat/test_resample.R | 4 ++-- 8 files changed, 18 insertions(+), 23 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index fdcc568d5..f6d246363 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -249,8 +249,6 @@ importFrom(R6,R6Class) importFrom(R6,is.R6) importFrom(RhpcBLASctl,blas_get_num_procs) importFrom(RhpcBLASctl,blas_set_num_threads) -importFrom(data.table,as.data.table) -importFrom(data.table,data.table) importFrom(future,nbrOfWorkers) importFrom(future,plan) importFrom(graphics,plot) diff --git a/R/Learner.R b/R/Learner.R index 09ccba6a0..e2030d84d 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -243,6 +243,7 @@ Learner = R6Class("Learner", test_row_ids = task$row_roles$test learner_train(learner, task, train_row_ids = train_row_ids, test_row_ids = test_row_ids, mode = mode) + self$model = unmarshal_model(self$state$model) # store data prototype proto = task$data(rows = integer()) @@ -571,3 +572,4 @@ default_values.Learner = function(x, search_space, task, ...) { # nolint # format_list_item.Learner = function(x, ...) { # nolint # sprintf("", x$id) # } + diff --git a/R/marshal.R b/R/marshal.R index 3bc2793a4..b68d2ad3b 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -76,13 +76,13 @@ learner_marshaled = function(learner) { #' @rdname marshaling #' @export -marshal_model = function(model, ...) { +marshal_model = function(model, clone, ...) { UseMethod("marshal_model") } #' @rdname marshaling #' @export -unmarshal_model = function(model, ...) { +unmarshal_model = function(model, clone, ...) { if (marshaled_model(model) && is.character(model$packages)) { require_namespaces(model$packages) } diff --git a/R/worker.R b/R/worker.R index 33af61fb2..459ac24d5 100644 --- a/R/worker.R +++ b/R/worker.R @@ -1,4 +1,4 @@ -learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NULL, mode = "train") { +learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NULL, mode = "train", unmarshal = TRUE) { # This wrapper calls learner$.train, and additionally performs some basic # checks that the training was successful. # Exceptions here are possibly encapsulated, so that they get captured @@ -18,7 +18,7 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL stopf("Learner '%s' on task '%s' returned NULL during internal %s()", learner$id, task$id, mode) } - if (learner$encapsulate[["train"]] == "callr" && "marshal" %in% learner$properties) { + if (learner$encapsulate[["train"]] == "callr") { model = marshal_model(model) } @@ -74,13 +74,6 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL # unmarshal_model does nothing if the model was not marshaled - - # We always want to unmarshal the model, because either: - # a) the user called $train() manually or - # b) we are within worker and still have to make a prediction - if ("marshal" %in% learner$properties) { - result$result = unmarshal_model(result$result) - } learner$state = insert_named(learner$state, list( model = result$result, log = log, @@ -266,7 +259,10 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, learner$param_set$set_values(.values = param_values) } learner_hash = learner$hash - learner = learner_train(learner, task, sets[["train"]], sets[["test"]], mode = mode) + learner = learner_train(learner, task, sets[["train"]], sets[["test"]], mode = mode, unmarshal = FALSE) + + model_marshaled = if (store_models) marshal_model(learner$model) + learner$model = unmarshal_model(learner$model) # predict for each set sets = sets[learner$predict_sets] @@ -281,9 +277,10 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, learner$state$model = NULL } else if ("marshal" %in% learner$properties && !is_sequential) { lg$debug("marshaling model for learner '%s'", learner$id) - learner$model = marshal_model(learner$model) + learner$model = model_marshaled } + list(learner_state = learner$state, prediction = pdatas, param_values = learner$param_set$values, learner_hash = learner_hash) } diff --git a/man/marshaling.Rd b/man/marshaling.Rd index fb4b6b50a..5908111d9 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -16,9 +16,9 @@ learner_marshal(learner) learner_marshaled(learner) -marshal_model(model, ...) +marshal_model(model, clone, ...) -unmarshal_model(model, ...) +unmarshal_model(model, clone, ...) marshaled_model(model) } diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index 2fbda9a0f..e15d9dc4f 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -501,7 +501,7 @@ test_that("sequential execution does not trigger marshaling", { expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 0) }) -test_that("parallel execution and callr marshal twice", { +test_that("parallel execution and callr marshal once", { learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) task = tsk("iris") resampling = rsmp("holdout") @@ -509,7 +509,7 @@ test_that("parallel execution and callr marshal twice", { bmr = with_future(future::multisession, { benchmark(design, store_models = TRUE, unmarshal = TRUE) }) - expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 2) + expect_equal(bmr$resample_result(1)$learners[[1]]$model$marshal_count, 1) expect_false(bmr$resample_result(1)$learners[[1]]$marshaled) }) diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R index 1b5d4633f..f9fd5b90f 100644 --- a/tests/testthat/test_marshal.R +++ b/tests/testthat/test_marshal.R @@ -33,5 +33,3 @@ test_that("unmarshaling a unmarshaled object does nothing", { expect_equal(unmarshal_model(xm), x) expect_equal(unmarshal_model(unmarshal_model(xm)), x) }) - -test_that("versions are required") diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index 6dd046614..ff571b0bd 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -178,14 +178,14 @@ test_that("sequential execution does not trigger marshaling", { expect_equal(rr$learners[[1]]$model$marshal_count, 0) }) -test_that("parallel execution and callr marshal twice", { +test_that("parallel execution and callr marshal once", { learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) task = tsk("iris") resampling = rsmp("holdout") rr = with_future(future::multisession, { resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) }) - expect_equal(rr$learners[[1]]$model$marshal_count, 2) + expect_equal(rr$learners[[1]]$model$marshal_count, 1) expect_false(rr$learners[[1]]$marshaled) }) From 432971f3fb3ee21102e97c0b9156a4583d62186f Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Thu, 22 Feb 2024 20:11:27 +0100 Subject: [PATCH 22/47] optimization --- NAMESPACE | 2 ++ R/Learner.R | 2 +- R/marshal.R | 4 ++-- R/worker.R | 22 ++++++++++++++++++---- tests/testthat/test_mlr_learners.R | 6 +++++- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index f6d246363..fdcc568d5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -249,6 +249,8 @@ importFrom(R6,R6Class) importFrom(R6,is.R6) importFrom(RhpcBLASctl,blas_get_num_procs) importFrom(RhpcBLASctl,blas_set_num_threads) +importFrom(data.table,as.data.table) +importFrom(data.table,data.table) importFrom(future,nbrOfWorkers) importFrom(future,plan) importFrom(graphics,plot) diff --git a/R/Learner.R b/R/Learner.R index e2030d84d..be1b5a41d 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -243,7 +243,7 @@ Learner = R6Class("Learner", test_row_ids = task$row_roles$test learner_train(learner, task, train_row_ids = train_row_ids, test_row_ids = test_row_ids, mode = mode) - self$model = unmarshal_model(self$state$model) + self$model = unmarshal_model(self$state$model, clone = FALSE) # store data prototype proto = task$data(rows = integer()) diff --git a/R/marshal.R b/R/marshal.R index b68d2ad3b..0d0f9cb38 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -48,7 +48,7 @@ learner_unmarshal = function(learner) { stopf("Cannot unmarshal, Learner '%s' has not been trained yet", learner$id) } # this will do nothing if the model was not marshaled - learner$model = unmarshal_model(learner$model) + learner$model = unmarshal_model(learner$model, clone = FALSE) invisible(learner) } @@ -60,7 +60,7 @@ learner_marshal = function(learner) { stopf("Cannot marshal, Learner '%s' has not been trained yet", learner$id) } # this will do nothing if the model was already marshaled - learner$model = marshal_model(learner$model) + learner$model = marshal_model(learner$model, clone = FALSE) invisible(learner) } diff --git a/R/worker.R b/R/worker.R index 459ac24d5..978c0f0e1 100644 --- a/R/worker.R +++ b/R/worker.R @@ -261,8 +261,17 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, learner_hash = learner$hash learner = learner_train(learner, task, sets[["train"]], sets[["test"]], mode = mode, unmarshal = FALSE) - model_marshaled = if (store_models) marshal_model(learner$model) - learner$model = unmarshal_model(learner$model) + model_marshaled = NULL + if (store_models && marshaled_model(learner$model) && !is_sequential) { + if (is_sequential) { + # callr + no parallelization + learner$model = unmarshal_model(learner$model, clone = FALSE) + } else { + # callr + parallelization + model_marshaled = learner$model + learner$model = unmarshal_model(learner$model, clone = TRUE) + } + } # predict for each set sets = sets[learner$predict_sets] @@ -275,10 +284,15 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, if (!store_models) { lg$debug("Erasing stored model for learner '%s'", learner$id) learner$state$model = NULL - } else if ("marshal" %in% learner$properties && !is_sequential) { - lg$debug("marshaling model for learner '%s'", learner$id) + } else if (!is.null(model_marshaled)) { + # callr + parallelization learner$model = model_marshaled + } else if (!is_sequential) { + # parallelization without callr + learner$model = marshal_model(learner$model, clone = FALSE) } + # no parallelization and no callr --> nothing to do + # no parallelization and callr --> already unmarshaled list(learner_state = learner$state, prediction = pdatas, param_values = learner$param_set$values, learner_hash = learner_hash) diff --git a/tests/testthat/test_mlr_learners.R b/tests/testthat/test_mlr_learners.R index 082a5527c..311cf654f 100644 --- a/tests/testthat/test_mlr_learners.R +++ b/tests/testthat/test_mlr_learners.R @@ -4,7 +4,11 @@ test_that("mlr_learners", { for (key in keys) { l = lrn(key) - expect_learner(l) + if (key == "classif.lily") { + expect_learner(l, task = tsk("iris")) + } else { + expect_learner(l) + } if (inherits(l, "TaskClassif")) { expect_true(startsWith(l$id, "classif.")) } From f9b33ea8192ed554236d8e2d96851562c73b3523 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Mon, 26 Feb 2024 13:58:38 +0100 Subject: [PATCH 23/47] add marshal property to regr.debug and remove lily --- DESCRIPTION | 1 - NAMESPACE | 7 +- R/LearnerClassifLily.R | 74 ------------- R/LearnerRegrDebug.R | 56 ++++++++-- R/marshal.R | 2 +- man/mlr_learners_classif.lily.Rd | 135 ----------------------- man/mlr_learners_regr.debug.Rd | 32 ++++++ tests/testthat/test_HotstartStack.R | 4 +- tests/testthat/test_Learner.R | 4 +- tests/testthat/test_LearnerClassifLily.R | 34 ------ tests/testthat/test_Measure.R | 2 +- tests/testthat/test_benchmark.R | 18 +-- tests/testthat/test_marshal.R | 40 ++++++- tests/testthat/test_mlr_learners.R | 4 +- tests/testthat/test_resample.R | 18 +-- 15 files changed, 146 insertions(+), 285 deletions(-) delete mode 100644 R/LearnerClassifLily.R delete mode 100644 man/mlr_learners_classif.lily.Rd delete mode 100644 tests/testthat/test_LearnerClassifLily.R diff --git a/DESCRIPTION b/DESCRIPTION index 8605d7aec..741659df0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -90,7 +90,6 @@ Collate: 'mlr_learners.R' 'LearnerClassifDebug.R' 'LearnerClassifFeatureless.R' - 'LearnerClassifLily.R' 'LearnerClassifRpart.R' 'LearnerRegr.R' 'LearnerRegrDebug.R' diff --git a/NAMESPACE b/NAMESPACE index fdcc568d5..59c85d75c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -87,9 +87,9 @@ S3method(fix_factor_levels,data.table) S3method(head,Task) S3method(is_missing_prediction_data,PredictionDataClassif) S3method(is_missing_prediction_data,PredictionDataRegr) -S3method(marshal_model,classif_lily_model) S3method(marshal_model,default) S3method(marshal_model,marshaled) +S3method(marshal_model,regr.debug_model) S3method(partition,Task) S3method(partition,TaskClassif) S3method(partition,TaskRegr) @@ -107,9 +107,9 @@ S3method(set_threads,default) S3method(set_threads,list) S3method(summary,Task) S3method(tail,Task) -S3method(unmarshal_model,classif_lily_model_marshaled) S3method(unmarshal_model,default) S3method(unmarshal_model,marshaled) +S3method(unmarshal_model,regr.debug_model_marshaled) export(BenchmarkResult) export(DataBackend) export(DataBackendDataTable) @@ -119,7 +119,6 @@ export(Learner) export(LearnerClassif) export(LearnerClassifDebug) export(LearnerClassifFeatureless) -export(LearnerClassifLily) export(LearnerClassifRpart) export(LearnerRegr) export(LearnerRegrDebug) @@ -249,8 +248,6 @@ importFrom(R6,R6Class) importFrom(R6,is.R6) importFrom(RhpcBLASctl,blas_get_num_procs) importFrom(RhpcBLASctl,blas_set_num_threads) -importFrom(data.table,as.data.table) -importFrom(data.table,data.table) importFrom(future,nbrOfWorkers) importFrom(future,plan) importFrom(graphics,plot) diff --git a/R/LearnerClassifLily.R b/R/LearnerClassifLily.R deleted file mode 100644 index 4f37e29d3..000000000 --- a/R/LearnerClassifLily.R +++ /dev/null @@ -1,74 +0,0 @@ -#' @title Lily and marshal -#' -#' @name mlr_learners_classif.lily -#' @include LearnerClassifDebug.R -#' -#' @description -#' This learner is just like [`LearnerClassifDebug`], but can be marshaled. -#' When the `count_marshaling` parameter is `TRUE`, the model contains a `marshal_count` that will be increased -#' by 1, each time `marshal_model` is called. -#' -#' @templateVar id classif.lily -#' @template learner -#' -#' @export -LearnerClassifLily = R6Class("LearnerClassifLily", - inherit = LearnerClassifDebug, - public = list( - #' @description - #' Creates a new instance of this [R6][R6::R6Class] class. - initialize = function() { - super$initialize() - self$param_set$add(ps(count_marshaling = p_lgl(tags = c("train", "required")))) - self$param_set$values$count_marshaling = FALSE - self$properties = sort(c("marshal", self$properties)) - self$man = "mlr3::mlr_learners_classif.lily" - self$label = "Lily Learner" - self$id = "classif.lily" - }, - #' @description - #' Marshals the learner. - marshal = function() { - learner_marshal(self) - }, - #' @description - #' Unmarshal the learner. - unmarshal = function() { - learner_unmarshal(self) - } - ), - active = list( - #' @field marshaled (logical(1))\cr - #' Whether the learner has been marshaled. - marshaled = function() { - learner_marshaled(self) - } - ), - private = list( - .train = function(task) { - model = super$.train(task) - if (self$param_set$values$count_marshaling) { - model$marshal_count = 0L - } - class(model) = "classif_lily_model" - return(model) - } - ) -) - -#' @include mlr_learners.R -mlr_learners$add("classif.lily", function() LearnerClassifLily$new()) - -#' @export -marshal_model.classif_lily_model = function(model, ...) { - if (!is.null(model$marshal_count)) { - model$marshal_count = model$marshal_count + 1 - } - newclass = c("classif_lily_model_marshaled", "marshaled") - structure(list(marshaled = model, packages = "mlr3"), class = newclass) -} - -#' @export -unmarshal_model.classif_lily_model_marshaled = function(model, ...) { - model$marshaled -} diff --git a/R/LearnerRegrDebug.R b/R/LearnerRegrDebug.R index 6cd6b6289..62d7231fa 100644 --- a/R/LearnerRegrDebug.R +++ b/R/LearnerRegrDebug.R @@ -13,6 +13,7 @@ #' \item{save_tasks:}{Saves input task in `model` slot during training and prediction.} #' \item{threads:}{Number of threads to use. Has no effect.} #' \item{x:}{Numeric tuning parameter. Has no effect.} +#' \item{count_marshaling:}{If `TRUE`, `marshal_model` will increase the `marshal_count` by 1. This value is initialized to `FALSE`.} #' } #' #' @templateVar id regr.debug @@ -33,25 +34,44 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, #' @description #' Creates a new instance of this [R6][R6::R6Class] class. initialize = function() { + param_set = ps( + predict_missing = p_dbl(0, 1, default = 0, tags = "predict"), + predict_missing_type = p_fct(c("na", "omit"), default = "na", tags = "predict"), + save_tasks = p_lgl(default = FALSE, tags = c("train", "predict")), + threads = p_int(1L, tags = c("train", "threads")), + x = p_dbl(0, 1, tags = "train"), + count_marshaling = p_lgl(tags = c("train", "required")) + ) + param_set$set_values(count_marshaling = FALSE) super$initialize( id = "regr.debug", feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), predict_types = c("response", "se"), - param_set = ps( - predict_missing = p_dbl(0, 1, default = 0, tags = "predict"), - predict_missing_type = p_fct(c("na", "omit"), default = "na", tags = "predict"), - save_tasks = p_lgl(default = FALSE, tags = c("train", "predict")), - threads = p_int(1L, tags = c("train", "threads")), - x = p_dbl(0, 1, tags = "train") - ), + param_set = param_set, properties = "missings", man = "mlr3::mlr_learners_regr.debug", data_formats = c("data.table", "Matrix"), label = "Debug Learner for Regression" ) + }, + #' @description + #' Marshals the learner. + marshal = function() { + learner_marshal(self) + }, + #' @description + #' Unmarshal the learner. + unmarshal = function() { + learner_unmarshal(self) + } + ), + active = list( + #' @field marshaled (logical(1))\cr + #' Whether the learner has been marshaled. + marshaled = function() { + learner_marshaled(self) } ), - private = list( .train = function(task) { pv = self$param_set$get_values(tags = "train") @@ -65,6 +85,9 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, if (isTRUE(pv$save_tasks)) { model$task_train = task$clone(deep = TRUE) } + if (isTRUE(pv$count_marshaling)) { + model$marshal_count = 0L + } set_class(model, "regr.debug_model") }, @@ -100,3 +123,20 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, #' @include mlr_learners.R mlr_learners$add("regr.debug", function() LearnerRegrDebug$new()) + + +#' @export +#' @method marshal_model regr.debug_model +marshal_model.regr.debug_model = function(model, ...) { + if (!is.null(model$marshal_count)) { + model$marshal_count = model$marshal_count + 1 + } + newclass = c("regr.debug_model_marshaled", "marshaled") + structure(list(marshaled = model, packages = "mlr3"), class = newclass) +} + +#' @export +#' @method unmarshal_model regr.debug_model_marshaled +unmarshal_model.regr.debug_model_marshaled = function(model, ...) { + model$marshaled +} diff --git a/R/marshal.R b/R/marshal.R index 0d0f9cb38..51134160f 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -36,7 +36,7 @@ #' You can verify whether you have correctly implemented marshaling by using the internal test helper #' `expect_marshalable_learner()`. This is also run by `expect_learner()` if a task is provided. #' -#' For a concrete example on how to implement marshaling, see [`LearnerClassifLily`]. +#' For a concrete example on how to implement marshaling, see [`LearnerRegrDebug`]. #' #' @param learner [`Learner`]\cr #' The learner. diff --git a/man/mlr_learners_classif.lily.Rd b/man/mlr_learners_classif.lily.Rd deleted file mode 100644 index 5125b14d3..000000000 --- a/man/mlr_learners_classif.lily.Rd +++ /dev/null @@ -1,135 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/LearnerClassifLily.R -\name{mlr_learners_classif.lily} -\alias{mlr_learners_classif.lily} -\alias{LearnerClassifLily} -\title{Lily and marshal} -\description{ -This learner is just like \code{\link{LearnerClassifDebug}}, but can be marshaled. -When the \code{count_marshaling} parameter is \code{TRUE}, the model contains a \code{marshal_count} that will be increased -by 1, each time \code{marshal_model} is called. -} -\section{Dictionary}{ - -This \link{Learner} can be instantiated via the \link[mlr3misc:Dictionary]{dictionary} \link{mlr_learners} or with the associated sugar function \code{\link[=lrn]{lrn()}}: - -\if{html}{\out{
}}\preformatted{mlr_learners$get("classif.lily") -lrn("classif.lily") -}\if{html}{\out{
}} -} - -\section{Meta Information}{ - -\itemize{ -\item Task type: \dQuote{classif} -\item Predict Types: \dQuote{response}, \dQuote{prob} -\item Feature Types: \dQuote{logical}, \dQuote{integer}, \dQuote{numeric}, \dQuote{character}, \dQuote{factor}, \dQuote{ordered} -\item Required Packages: \CRANpkg{mlr3} -} -} - -\section{Parameters}{ -\tabular{lllll}{ - Id \tab Type \tab Default \tab Levels \tab Range \cr - error_predict \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - error_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - message_predict \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - message_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - predict_missing \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - predict_missing_type \tab character \tab na \tab na, omit \tab - \cr - save_tasks \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr - segfault_predict \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - segfault_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - sleep_train \tab untyped \tab - \tab \tab - \cr - sleep_predict \tab untyped \tab - \tab \tab - \cr - threads \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr - warning_predict \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - warning_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - x \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - iter \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr - count_marshaling \tab logical \tab - \tab TRUE, FALSE \tab - \cr -} -} - -\section{Super classes}{ -\code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerClassif]{mlr3::LearnerClassif}} -> \code{\link[mlr3:LearnerClassifDebug]{mlr3::LearnerClassifDebug}} -> \code{LearnerClassifLily} -} -\section{Active bindings}{ -\if{html}{\out{
}} -\describe{ -\item{\code{marshaled}}{(logical(1))\cr -Whether the learner has been marshaled.} -} -\if{html}{\out{
}} -} -\section{Methods}{ -\subsection{Public methods}{ -\itemize{ -\item \href{#method-LearnerClassifLily-new}{\code{LearnerClassifLily$new()}} -\item \href{#method-LearnerClassifLily-marshal}{\code{LearnerClassifLily$marshal()}} -\item \href{#method-LearnerClassifLily-unmarshal}{\code{LearnerClassifLily$unmarshal()}} -\item \href{#method-LearnerClassifLily-clone}{\code{LearnerClassifLily$clone()}} -} -} -\if{html}{\out{ -
Inherited methods - -
-}} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerClassifLily-new}{}}} -\subsection{Method \code{new()}}{ -Creates a new instance of this \link[R6:R6Class]{R6} class. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerClassifLily$new()}\if{html}{\out{
}} -} - -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerClassifLily-marshal}{}}} -\subsection{Method \code{marshal()}}{ -Marshals the learner. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerClassifLily$marshal()}\if{html}{\out{
}} -} - -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerClassifLily-unmarshal}{}}} -\subsection{Method \code{unmarshal()}}{ -Unmarshal the learner. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerClassifLily$unmarshal()}\if{html}{\out{
}} -} - -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerClassifLily-clone}{}}} -\subsection{Method \code{clone()}}{ -The objects of this class are cloneable with this method. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerClassifLily$clone(deep = FALSE)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{deep}}{Whether to make a deep clone.} -} -\if{html}{\out{
}} -} -} -} diff --git a/man/mlr_learners_regr.debug.Rd b/man/mlr_learners_regr.debug.Rd index 2c2c53b23..4cb21798f 100644 --- a/man/mlr_learners_regr.debug.Rd +++ b/man/mlr_learners_regr.debug.Rd @@ -14,6 +14,7 @@ The following hyperparameters trigger the following actions: \item{save_tasks:}{Saves input task in \code{model} slot during training and prediction.} \item{threads:}{Number of threads to use. Has no effect.} \item{x:}{Numeric tuning parameter. Has no effect.} +\item{count_marshaling:}{If \code{TRUE}, \code{marshal_model} will increase the \code{marshal_count} by 1. This value is initialized to \code{FALSE}.} } } \section{Dictionary}{ @@ -43,6 +44,7 @@ lrn("regr.debug") save_tasks \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr threads \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr x \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + count_marshaling \tab logical \tab - \tab TRUE, FALSE \tab - \cr } } @@ -89,10 +91,20 @@ Other Learner: \section{Super classes}{ \code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrDebug} } +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{marshaled}}{(logical(1))\cr +Whether the learner has been marshaled.} +} +\if{html}{\out{
}} +} \section{Methods}{ \subsection{Public methods}{ \itemize{ \item \href{#method-LearnerRegrDebug-new}{\code{LearnerRegrDebug$new()}} +\item \href{#method-LearnerRegrDebug-marshal}{\code{LearnerRegrDebug$marshal()}} +\item \href{#method-LearnerRegrDebug-unmarshal}{\code{LearnerRegrDebug$unmarshal()}} \item \href{#method-LearnerRegrDebug-clone}{\code{LearnerRegrDebug$clone()}} } } @@ -119,6 +131,26 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. \if{html}{\out{
}}\preformatted{LearnerRegrDebug$new()}\if{html}{\out{
}} } +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrDebug-marshal}{}}} +\subsection{Method \code{marshal()}}{ +Marshals the learner. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrDebug$marshal()}\if{html}{\out{
}} +} + +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerRegrDebug-unmarshal}{}}} +\subsection{Method \code{unmarshal()}}{ +Unmarshal the learner. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerRegrDebug$unmarshal()}\if{html}{\out{
}} +} + } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/tests/testthat/test_HotstartStack.R b/tests/testthat/test_HotstartStack.R index 26408e4fc..abe1bb14d 100644 --- a/tests/testthat/test_HotstartStack.R +++ b/tests/testthat/test_HotstartStack.R @@ -412,8 +412,8 @@ test_that("HotstartStack threshold works", { test_that("error when adding marshaled learner", { hot = HotstartStack$new() - learner = lrn("classif.lily") - learner$train(tsk("iris")) + learner = lrn("regr.debuLg") + learner$train(tsk("mtcars")) learner$marshal() expect_error(hot$add(learner), "unmarshaled") }) diff --git a/tests/testthat/test_Learner.R b/tests/testthat/test_Learner.R index 41e7050a6..519788976 100644 --- a/tests/testthat/test_Learner.R +++ b/tests/testthat/test_Learner.R @@ -326,8 +326,8 @@ test_that("Models can be replaced", { }) test_that("marshaling and encapsulation", { - task = tsk("iris") - learner = lrn("classif.lily", count_marshaling = TRUE) + task = tsk("mtcars") + learner = lrn("regr.debug", count_marshaling = TRUE) # callr encapsulation causes marshaling learner$encapsulate = c(train = "callr") diff --git a/tests/testthat/test_LearnerClassifLily.R b/tests/testthat/test_LearnerClassifLily.R deleted file mode 100644 index 9d35ef495..000000000 --- a/tests/testthat/test_LearnerClassifLily.R +++ /dev/null @@ -1,34 +0,0 @@ -test_that("lily", { - task = tsk("iris") - learner = lrn("classif.lily") - learner$train(task) - expect_false(learner$marshaled) - learner$marshal() - expect_true(learner$marshaled) - expect_error(learner$predict(task), "has not been unmarshaled") - learner$unmarshal() - expect_learner(learner, task) -}) - -test_that("marshal count works", { - # to mock that marshaling behaves as expected, we need to be know how often it happened - # note that this means that marshal_model modifies the model in a permanent way, i.e. this is not reversed by - # unmarshal_model. - learner = lrn("classif.lily", count_marshaling = TRUE) - task = tsk("iris") - learner$train(task) - expect_equal(learner$model$marshal_count, 0) - learner$marshal()$unmarshal() - expect_equal(learner$model$marshal_count, 1) - learner$marshal()$unmarshal() - expect_equal(learner$model$marshal_count, 2) - - # TO make the lily learner more realistic (i.e. (un)marshaling leaves the object unchanged) - # the count_marshaling parameter can also be set to FALSE - learner2 = lrn("classif.lily", count_marshaling = FALSE) - learner2$train(task) - expect_true(is.null(learner2$model$marshal_count)) - model1 = learner2$model - model2 = learner2$marshal()$unmarshal()$model - expect_equal(model1, model2) -}) diff --git a/tests/testthat/test_Measure.R b/tests/testthat/test_Measure.R index 5acf1e6eb..a29a08639 100644 --- a/tests/testthat/test_Measure.R +++ b/tests/testthat/test_Measure.R @@ -135,7 +135,7 @@ test_that("scoring fails when measure requires_model, but model is in marshaled measure = msr("classif.acc") measure$properties = c(measure$properties, "requires_model") - rr = resample(tsk("iris"), lrn("classif.lily"), rsmp("holdout")) + rr = resample(tsk("mtcars"), lrn("regr.debug"), rsmp("holdout")) rr$marshal() rr$score(msr("selected_features")) diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index e15d9dc4f..bf55952c1 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -479,8 +479,8 @@ test_that("param_values in benchmark", { test_that("parallel execution automatically triggers marshaling", { - learner = lrn("classif.lily", count_marshaling = TRUE) - task = tsk("iris") + learner = lrn("regr.debug", count_marshaling = TRUE) + task = tsk("mtcars") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) bmr = with_future(future::multisession, { @@ -491,8 +491,8 @@ test_that("parallel execution automatically triggers marshaling", { }) test_that("sequential execution does not trigger marshaling", { - learner = lrn("classif.lily", count_marshaling = TRUE) - task = tsk("iris") + learner = lrn("regr.debug", count_marshaling = TRUE) + task = tsk("mtcars") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) bmr = with_future(future::sequential, { @@ -502,8 +502,8 @@ test_that("sequential execution does not trigger marshaling", { }) test_that("parallel execution and callr marshal once", { - learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) - task = tsk("iris") + learner = lrn("regr.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + task = tsk("mtcars") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) bmr = with_future(future::multisession, { @@ -515,8 +515,8 @@ test_that("parallel execution and callr marshal once", { test_that("unmarshal parameter is respected", { - learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) - task = tsk("iris") + learner = lrn("regr.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + task = tsk("mtcars") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) bmr = with_future(future::multisession, { @@ -530,7 +530,7 @@ test_that("unmarshal parameter is respected", { }) test_that("BenchmarkResult can be (un)marshaled", { - bmr = benchmark(benchmark_grid(tsk("iris"), lrn("classif.lily"), rsmp("holdout")), store_models = TRUE) + bmr = benchmark(benchmark_grid(tsk("mtcars"), lrn("regr.debug"), rsmp("holdout")), store_models = TRUE) expect_false(bmr$resample_result(1)$learners[[1]]$marshaled) bmr$marshal() expect_true(bmr$resample_result(1)$learners[[1]]$marshaled) diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R index f9fd5b90f..3224ac385 100644 --- a/tests/testthat/test_marshal.R +++ b/tests/testthat/test_marshal.R @@ -1,6 +1,6 @@ test_that("learner methods", { - learner = lrn("classif.lily") - task = tsk("iris") + learner = lrn("regr.debug") + task = tsk("mtcars") expect_error(learner_marshal(learner), "not been trained") expect_error(learner_unmarshal(learner), "not been trained") expect_error(learner_marshaled(learner), "not been trained") @@ -33,3 +33,39 @@ test_that("unmarshaling a unmarshaled object does nothing", { expect_equal(unmarshal_model(xm), x) expect_equal(unmarshal_model(unmarshal_model(xm)), x) }) + + +test_that("marshaling for LearnerRegrDebug", { + task = tsk("mtcars") + learner = lrn("regr.debug") + learner$train(task) + expect_false(learner$marshaled) + learner$marshal() + expect_true(learner$marshaled) + expect_error(learner$predict(task), "has not been unmarshaled") + learner$unmarshal() + expect_learner(learner, task) +}) + +test_that("marshal count works for LearnerRegrDebug", { + # to mock that marshaling behaves as expected, we need to be know how often it happened + # note that this means that marshal_model modifies the model in a permanent way, i.e. this is not reversed by + # unmarshal_model. + learner = lrn("regr.debug", count_marshaling = TRUE) + task = tsk("mtcars") + learner$train(task) + expect_equal(learner$model$marshal_count, 0) + learner$marshal()$unmarshal() + expect_equal(learner$model$marshal_count, 1) + learner$marshal()$unmarshal() + expect_equal(learner$model$marshal_count, 2) + + # TO make the lily learner more realistic (i.e. (un)marshaling leaves the object unchanged) + # the count_marshaling parameter can also be set to FALSE + learner2 = lrn("regr.debug", count_marshaling = FALSE) + learner2$train(task) + expect_true(is.null(learner2$model$marshal_count)) + model1 = learner2$model + model2 = learner2$marshal()$unmarshal()$model + expect_equal(model1, model2) +}) diff --git a/tests/testthat/test_mlr_learners.R b/tests/testthat/test_mlr_learners.R index 311cf654f..9ab2ab74b 100644 --- a/tests/testthat/test_mlr_learners.R +++ b/tests/testthat/test_mlr_learners.R @@ -4,8 +4,8 @@ test_that("mlr_learners", { for (key in keys) { l = lrn(key) - if (key == "classif.lily") { - expect_learner(l, task = tsk("iris")) + if (key == "regr.debug") { + expect_learner(l, task = tsk("mtcars")) } else { expect_learner(l) } diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index ff571b0bd..521585164 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -158,8 +158,8 @@ test_that("as_resample_result works for result data", { }) test_that("parallel execution automatically triggers marshaling", { - learner = lrn("classif.lily", count_marshaling = TRUE) - task = tsk("iris") + learner = lrn("regr.debug", count_marshaling = TRUE) + task = tsk("mtcars") resampling = rsmp("holdout") rr = with_future(future::multisession, { resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) @@ -169,8 +169,8 @@ test_that("parallel execution automatically triggers marshaling", { }) test_that("sequential execution does not trigger marshaling", { - learner = lrn("classif.lily", count_marshaling = TRUE) - task = tsk("iris") + learner = lrn("regr.debug", count_marshaling = TRUE) + task = tsk("mtcars") resampling = rsmp("holdout") rr = with_future(future::sequential, { resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) @@ -179,8 +179,8 @@ test_that("sequential execution does not trigger marshaling", { }) test_that("parallel execution and callr marshal once", { - learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) - task = tsk("iris") + learner = lrn("regr.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + task = tsk("mtcars") resampling = rsmp("holdout") rr = with_future(future::multisession, { resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) @@ -191,8 +191,8 @@ test_that("parallel execution and callr marshal once", { test_that("unmarshal parameter is respected", { - learner = lrn("classif.lily", count_marshaling = TRUE, encapsulate = c(train = "callr")) - task = tsk("iris") + learner = lrn("regr.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + task = tsk("mtcars") resampling = rsmp("holdout") rr = with_future(future::multisession, { list( @@ -205,7 +205,7 @@ test_that("unmarshal parameter is respected", { }) test_that("ResampleResult can be (un)marshaled", { - rr = resample(tsk("iris"), lrn("classif.lily"), rsmp("holdout"), store_models = TRUE) + rr = resample(tsk("mtcars"), lrn("regr.debug"), rsmp("holdout"), store_models = TRUE) expect_false(rr$learners[[1]]$marshaled) rr$marshal() expect_true(rr$learners[[1]]$marshaled) From d6ceb1c197c08a322aca69aeca363ac01f864080 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Mon, 4 Mar 2024 17:20:24 +0100 Subject: [PATCH 24/47] inplace --- R/marshal.R | 8 ++++---- R/worker.R | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/R/marshal.R b/R/marshal.R index 51134160f..95d8e835d 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -48,7 +48,7 @@ learner_unmarshal = function(learner) { stopf("Cannot unmarshal, Learner '%s' has not been trained yet", learner$id) } # this will do nothing if the model was not marshaled - learner$model = unmarshal_model(learner$model, clone = FALSE) + learner$model = unmarshal_model(learner$model, inplace = TRUE) invisible(learner) } @@ -60,7 +60,7 @@ learner_marshal = function(learner) { stopf("Cannot marshal, Learner '%s' has not been trained yet", learner$id) } # this will do nothing if the model was already marshaled - learner$model = marshal_model(learner$model, clone = FALSE) + learner$model = marshal_model(learner$model, inplace = TRUE) invisible(learner) } @@ -76,13 +76,13 @@ learner_marshaled = function(learner) { #' @rdname marshaling #' @export -marshal_model = function(model, clone, ...) { +marshal_model = function(model, inplace, ...) { UseMethod("marshal_model") } #' @rdname marshaling #' @export -unmarshal_model = function(model, clone, ...) { +unmarshal_model = function(model, inplace, ...) { if (marshaled_model(model) && is.character(model$packages)) { require_namespaces(model$packages) } diff --git a/R/worker.R b/R/worker.R index 978c0f0e1..81fdb45fa 100644 --- a/R/worker.R +++ b/R/worker.R @@ -19,7 +19,7 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL } if (learner$encapsulate[["train"]] == "callr") { - model = marshal_model(model) + model = marshal_model(model, inplace = TRUE) } model From cdf603f7a63166a9d36ddd1f5547922f80d3442d Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 5 Mar 2024 17:41:54 +0100 Subject: [PATCH 25/47] some more fixes --- NAMESPACE | 2 ++ R/BenchmarkResult.R | 14 +++++---- R/LearnerRegrDebug.R | 12 +++++--- R/ResampleResult.R | 16 ++++++---- R/ResultData.R | 12 +++++--- R/marshal.R | 46 ++++++++++++++--------------- man/BenchmarkResult.Rd | 22 ++++++++++++-- man/ResampleResult.Rd | 24 ++++++++++++--- man/ResultData.Rd | 20 +++++++++++-- man/marshaling.Rd | 14 ++++----- man/mlr_learners_regr.debug.Rd | 20 +++++++++++-- tests/testthat/test_HotstartStack.R | 2 +- 12 files changed, 143 insertions(+), 61 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 59c85d75c..322efffce 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -248,6 +248,8 @@ importFrom(R6,R6Class) importFrom(R6,is.R6) importFrom(RhpcBLASctl,blas_get_num_procs) importFrom(RhpcBLASctl,blas_set_num_threads) +importFrom(data.table,as.data.table) +importFrom(data.table,data.table) importFrom(future,nbrOfWorkers) importFrom(future,plan) importFrom(graphics,plot) diff --git a/R/BenchmarkResult.R b/R/BenchmarkResult.R index f33290f9b..b3d446257 100644 --- a/R/BenchmarkResult.R +++ b/R/BenchmarkResult.R @@ -134,14 +134,18 @@ BenchmarkResult = R6Class("BenchmarkResult", }, #' @description - #' marshals all stored models. - marshal = function() { - private$.data$marshal() + #' Marshals all stored models. + #' @param ... (any)\cr + #' Additional arguments passed to [`marshal_model()`]. + marshal = function(...) { + private$.data$marshal(...) }, #' @description #' Unmarshals all stored models. - unmarshal = function() { - private$.data$unmarshal() + #' @param ... (any)\cr + #' Additional arguments passed to [`unmarshal_model()`]. + unmarshal = function(...) { + private$.data$unmarshal(...) }, #' @description diff --git a/R/LearnerRegrDebug.R b/R/LearnerRegrDebug.R index 62d7231fa..11551da73 100644 --- a/R/LearnerRegrDebug.R +++ b/R/LearnerRegrDebug.R @@ -56,13 +56,17 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, }, #' @description #' Marshals the learner. - marshal = function() { - learner_marshal(self) + #' @param ... (any)\cr + #' Additional arguments passed to [`marshal_model()`]. + marshal = function(...) { + learner_marshal(.learner = self, ...) }, #' @description #' Unmarshal the learner. - unmarshal = function() { - learner_unmarshal(self) + #' @param ... (any)\cr + #' Additional arguments passed to [`unmarshal_model()`]. + unmarshal = function(...) { + learner_unmarshal(.learner = self, ...) } ), active = list( diff --git a/R/ResampleResult.R b/R/ResampleResult.R index fc15d4d4c..479e54e30 100644 --- a/R/ResampleResult.R +++ b/R/ResampleResult.R @@ -225,14 +225,18 @@ ResampleResult = R6Class("ResampleResult", }, #' @description - #' marshals all stored learner models. - marshal = function() { - private$.data$marshal() + #' Marshals all stored models. + #' @param ... (any)\cr + #' Additional arguments passed to [`marshal_model()`]. + marshal = function(...) { + private$.data$marshal(...) }, #' @description - #' Unmarshals all stored learner models. - unmarshal = function() { - private$.data$unmarshal() + #' Unmarshals all stored models. + #' @param ... (any)\cr + #' Additional arguments passed to [`unmarshal_model()`]. + unmarshal = function(...) { + private$.data$unmarshal(...) } ), diff --git a/R/ResultData.R b/R/ResultData.R index 5a51acdab..8d7828547 100644 --- a/R/ResultData.R +++ b/R/ResultData.R @@ -245,17 +245,21 @@ ResultData = R6Class("ResultData", #' @description #' Marshals all stored learner models. #' This will do nothing to models that are already marshaled. - marshal = function() { + #' @param ... (any)\cr + #' Additional arguments passed to [`marshal_model()`]. + marshal = function(...) { learner_state = NULL - self$data$fact[, learner_state := lapply(learner_state, marshal_state_if_model)] + self$data$fact[, learner_state := lapply(learner_state, function(x) marshal_state_if_model(.state = x, ...))] invisible(self) }, #' @description #' Unmarshals all stored learner models. #' This will do nothing to models which are not marshaled. - unmarshal = function() { + #' @param ... (any)\cr + #' Additional arguments passed to [`unmarshal_model()`]. + unmarshal = function(...) { learner_state = NULL - self$data$fact[, learner_state := lapply(learner_state, unmarshal_state_if_model)] + self$data$fact[, learner_state := lapply(learner_state, function(x) unmarshal_state_if_model(.state = x, ...))] invisible(self) }, diff --git a/R/marshal.R b/R/marshal.R index 95d8e835d..3361c24f8 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -38,40 +38,40 @@ #' #' For a concrete example on how to implement marshaling, see [`LearnerRegrDebug`]. #' -#' @param learner [`Learner`]\cr +#' @param .learner [`Learner`]\cr #' The learner. #' @keywords internal #' @export -learner_unmarshal = function(learner) { +learner_unmarshal = function(.learner, ...) { # no need to check for 'marshal' property as this method should only be available for such learners - if (is.null(learner$model)) { - stopf("Cannot unmarshal, Learner '%s' has not been trained yet", learner$id) + if (is.null(.learner$model)) { + stopf("Cannot unmarshal, Learner '%s' has not been trained yet", .learner$id) } # this will do nothing if the model was not marshaled - learner$model = unmarshal_model(learner$model, inplace = TRUE) - invisible(learner) + .learner$model = unmarshal_model(.learner$model, inplace = TRUE, ...) + invisible(.learner) } #' @rdname marshaling #' @export -learner_marshal = function(learner) { +learner_marshal = function(.learner, ...) { # no need to check for 'marshal' property as this method should only be available for such learners - if (is.null(learner$model)) { - stopf("Cannot marshal, Learner '%s' has not been trained yet", learner$id) + if (is.null(.learner$model)) { + stopf("Cannot marshal, Learner '%s' has not been trained yet", .learner$id) } # this will do nothing if the model was already marshaled - learner$model = marshal_model(learner$model, inplace = TRUE) - invisible(learner) + .learner$model = marshal_model(.learner$model, inplace = TRUE, ...) + invisible(.learner) } #' @rdname marshaling #' @export -learner_marshaled = function(learner) { +learner_marshaled = function(.learner) { # no need to check for 'marshal' property as this method should only be available for such learners - if (is.null(learner$model)) { - stopf("Cannot check marshaled status, Learner '%s' has not been trained yet", learner$id) + if (is.null(.learner$model)) { + stopf("Cannot check marshaled status, Learner '%s' has not been trained yet", .learner$id) } - marshaled_model(learner$model) + marshaled_model(.learner$model) } #' @rdname marshaling @@ -116,16 +116,16 @@ unmarshal_model.default = function(model, ...) { model } -marshal_state_if_model = function(state, ...) { - if (!is.null(state$model)) { - state$model = marshal_model(state$model) +marshal_state_if_model = function(.state, ...) { + if (!is.null(.state$model)) { + .state$model = marshal_model(.state$model, ...) } - state + .state } -unmarshal_state_if_model = function(state, ...) { - if (!is.null(state$model)) { - state$model = unmarshal_model(state$model) +unmarshal_state_if_model = function(.state, ...) { + if (!is.null(.state$model)) { + .state$model = unmarshal_model(.state$model, ...) } - state + .state } diff --git a/man/BenchmarkResult.Rd b/man/BenchmarkResult.Rd index 41d42d1a9..0ac33c0c1 100644 --- a/man/BenchmarkResult.Rd +++ b/man/BenchmarkResult.Rd @@ -236,11 +236,19 @@ the object in its previous state. \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-BenchmarkResult-marshal}{}}} \subsection{Method \code{marshal()}}{ -marshals all stored models. +Marshals all stored models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{BenchmarkResult$marshal()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{BenchmarkResult$marshal(...)}\if{html}{\out{
}} } +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=marshal_model]{marshal_model()}}.} +} +\if{html}{\out{
}} +} } \if{html}{\out{
}} \if{html}{\out{}} @@ -248,9 +256,17 @@ marshals all stored models. \subsection{Method \code{unmarshal()}}{ Unmarshals all stored models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{BenchmarkResult$unmarshal()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{BenchmarkResult$unmarshal(...)}\if{html}{\out{
}} } +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=unmarshal_model]{unmarshal_model()}}.} +} +\if{html}{\out{
}} +} } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/ResampleResult.Rd b/man/ResampleResult.Rd index 3e5e4e08d..8e4d93964 100644 --- a/man/ResampleResult.Rd +++ b/man/ResampleResult.Rd @@ -354,21 +354,37 @@ the object in its previous state. \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResampleResult-marshal}{}}} \subsection{Method \code{marshal()}}{ -marshals all stored learner models. +Marshals all stored models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResampleResult$marshal()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{ResampleResult$marshal(...)}\if{html}{\out{
}} } +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=marshal_model]{marshal_model()}}.} +} +\if{html}{\out{
}} +} } \if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-ResampleResult-unmarshal}{}}} \subsection{Method \code{unmarshal()}}{ -Unmarshals all stored learner models. +Unmarshals all stored models. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResampleResult$unmarshal()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{ResampleResult$unmarshal(...)}\if{html}{\out{
}} } +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=unmarshal_model]{unmarshal_model()}}.} +} +\if{html}{\out{
}} +} } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/ResultData.Rd b/man/ResultData.Rd index d276815cf..ee9ec2b82 100644 --- a/man/ResultData.Rd +++ b/man/ResultData.Rd @@ -340,9 +340,17 @@ Modified \code{self} (invisibly). Marshals all stored learner models. This will do nothing to models that are already marshaled. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultData$marshal()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{ResultData$marshal(...)}\if{html}{\out{
}} } +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=marshal_model]{marshal_model()}}.} +} +\if{html}{\out{
}} +} } \if{html}{\out{
}} \if{html}{\out{}} @@ -351,9 +359,17 @@ This will do nothing to models that are already marshaled. Unmarshals all stored learner models. This will do nothing to models which are not marshaled. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{ResultData$unmarshal()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{ResultData$unmarshal(...)}\if{html}{\out{
}} } +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=unmarshal_model]{unmarshal_model()}}.} +} +\if{html}{\out{
}} +} } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/marshaling.Rd b/man/marshaling.Rd index 5908111d9..ea2eb504c 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -10,20 +10,20 @@ \alias{marshaled_model} \title{(Un)marshal a Learner} \usage{ -learner_unmarshal(learner) +learner_unmarshal(.learner, ...) -learner_marshal(learner) +learner_marshal(.learner, ...) -learner_marshaled(learner) +learner_marshaled(.learner) -marshal_model(model, clone, ...) +marshal_model(model, inplace, ...) -unmarshal_model(model, clone, ...) +unmarshal_model(model, inplace, ...) marshaled_model(model) } \arguments{ -\item{learner}{\code{\link{Learner}}\cr +\item{.learner}{\code{\link{Learner}}\cr The learner.} } \description{ @@ -62,6 +62,6 @@ All three functions throw an error if the learner is not trained and otherwise c You can verify whether you have correctly implemented marshaling by using the internal test helper \code{expect_marshalable_learner()}. This is also run by \code{expect_learner()} if a task is provided. -For a concrete example on how to implement marshaling, see \code{\link{LearnerClassifLily}}. +For a concrete example on how to implement marshaling, see \code{\link{LearnerRegrDebug}}. } \keyword{internal} diff --git a/man/mlr_learners_regr.debug.Rd b/man/mlr_learners_regr.debug.Rd index 4cb21798f..d6c186457 100644 --- a/man/mlr_learners_regr.debug.Rd +++ b/man/mlr_learners_regr.debug.Rd @@ -138,9 +138,17 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. \subsection{Method \code{marshal()}}{ Marshals the learner. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrDebug$marshal()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{LearnerRegrDebug$marshal(...)}\if{html}{\out{
}} } +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=marshal_model]{marshal_model()}}.} +} +\if{html}{\out{
}} +} } \if{html}{\out{
}} \if{html}{\out{}} @@ -148,9 +156,17 @@ Marshals the learner. \subsection{Method \code{unmarshal()}}{ Unmarshal the learner. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrDebug$unmarshal()}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{LearnerRegrDebug$unmarshal(...)}\if{html}{\out{
}} } +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=unmarshal_model]{unmarshal_model()}}.} +} +\if{html}{\out{
}} +} } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/tests/testthat/test_HotstartStack.R b/tests/testthat/test_HotstartStack.R index abe1bb14d..7d6766f7c 100644 --- a/tests/testthat/test_HotstartStack.R +++ b/tests/testthat/test_HotstartStack.R @@ -412,7 +412,7 @@ test_that("HotstartStack threshold works", { test_that("error when adding marshaled learner", { hot = HotstartStack$new() - learner = lrn("regr.debuLg") + learner = lrn("regr.debug") learner$train(tsk("mtcars")) learner$marshal() expect_error(hot$add(learner), "unmarshaled") From ea2d75c45b4940da30654d7b288a54f9be28fba8 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 9 Apr 2024 12:56:21 +0200 Subject: [PATCH 26/47] rename --- NAMESPACE | 2 +- R/HotstartStack.R | 2 +- R/Learner.R | 4 ++-- R/Measure.R | 2 +- R/marshal.R | 10 +++++----- R/worker.R | 2 +- inst/testthat/helper_expectations.R | 4 ++-- man/marshaling.Rd | 8 ++++---- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 322efffce..8f3dfa130 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -213,6 +213,7 @@ export(default_measures) export(extract_pkgs) export(filter_prediction_data) export(install_pkgs) +export(is_marshaled_model) export(is_missing_prediction_data) export(learner_marshal) export(learner_marshaled) @@ -220,7 +221,6 @@ export(learner_unmarshal) export(lrn) export(lrns) export(marshal_model) -export(marshaled_model) export(mlr_learners) export(mlr_measures) export(mlr_reflections) diff --git a/R/HotstartStack.R b/R/HotstartStack.R index 7d2b40f36..a840daf4b 100644 --- a/R/HotstartStack.R +++ b/R/HotstartStack.R @@ -88,7 +88,7 @@ HotstartStack = R6Class("HotstartStack", walk(learners, function(learner) { if (is.null(learner$model)) { stopf("Learners must be trained before adding them to the hotstart stack.") - } else if (marshaled_model(learner$model)) { + } else if (is_marshaled_model(learner$model)) { stopf("Learners must be unmarshaled before adding them to the hotstart stack.") } }) diff --git a/R/Learner.R b/R/Learner.R index be1b5a41d..66b2dc90c 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -184,7 +184,7 @@ Learner = R6Class("Learner", #' @param ... (ignored). print = function(...) { catn(format(self), if (is.null(self$label) || is.na(self$label)) "" else paste0(": ", self$label)) - catn(str_indent("* Model:", if (is.null(self$model)) "-" else if (marshaled_model(self$model)) "" else paste0(class(self$model)[1L]))) + catn(str_indent("* Model:", if (is.null(self$model)) "-" else if (is_marshaled_model(self$model)) "" else paste0(class(self$model)[1L]))) catn(str_indent("* Parameters:", as_short_string(self$param_set$values, 1000L))) catn(str_indent("* Packages:", self$packages)) catn(str_indent("* Predict Types: ", replace(self$predict_types, self$predict_types == self$predict_type, paste0("[", self$predict_type, "]")))) @@ -280,7 +280,7 @@ Learner = R6Class("Learner", stopf("Cannot predict, Learner '%s' has not been trained yet", self$id) } - if (marshaled_model(self$model)) { + if (is_marshaled_model(self$model)) { stopf("Cannot predict, Learner '%s' has not been unmarshaled yet", self$id) } diff --git a/R/Measure.R b/R/Measure.R index 14acc2eea..e0c730954 100644 --- a/R/Measure.R +++ b/R/Measure.R @@ -185,7 +185,7 @@ Measure = R6Class("Measure", if ("requires_model" %in% self$properties && (is.null(learner) || is.null(learner$model))) { stopf("Measure '%s' requires the trained model", self$id) } - if ("requires_model" %in% self$properties && marshaled_model(learner$model)) { + if ("requires_model" %in% self$properties && is_marshaled_model(learner$model)) { stopf("Measure '%s' requires the trained model, but model is in marshaled form", self$id) } diff --git a/R/marshal.R b/R/marshal.R index 3361c24f8..bda18830c 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -20,7 +20,7 @@ #' Which takes in the marshaled model and returns it in unmarshaled form. #' The generic takes care that the packages specified during `"marshal"` are loaded, and errs if they are not. #' The returned object must not inherit from class `"marshaled"`. -#' * the function `marshaled_model(model)`, which returns `TRUE` if the model inherits from class `"marshaled"` +#' * the function `is_ marshaled_model(model)`, which returns `TRUE` if the model inherits from class `"marshaled"` #' and `FALSE` otherwise. #' #' In order to implement marshaling for a Learner, you only need to overload the `marshal_model` and `unmarshal_model` @@ -31,7 +31,7 @@ #' To make this as convenient as possible, the functions `learner_marshal(learner)`, `learner_unmarshal(learner)` #' and `learner_marshaled(learner)` are provided and can be called from within the public methods. #' All three functions throw an error if the learner is not trained and otherwise call -#' `marshal_model()`, `unmarshal_model()` or `marshaled_model()` on the learner's model. +#' `marshal_model()`, `unmarshal_model()` or `is_marshaled_model()` on the learner's model. #' #' You can verify whether you have correctly implemented marshaling by using the internal test helper #' `expect_marshalable_learner()`. This is also run by `expect_learner()` if a task is provided. @@ -71,7 +71,7 @@ learner_marshaled = function(.learner) { if (is.null(.learner$model)) { stopf("Cannot check marshaled status, Learner '%s' has not been trained yet", .learner$id) } - marshaled_model(.learner$model) + is_marshaled_model(.learner$model) } #' @rdname marshaling @@ -83,7 +83,7 @@ marshal_model = function(model, inplace, ...) { #' @rdname marshaling #' @export unmarshal_model = function(model, inplace, ...) { - if (marshaled_model(model) && is.character(model$packages)) { + if (is_marshaled_model(model) && is.character(model$packages)) { require_namespaces(model$packages) } UseMethod("unmarshal_model") @@ -91,7 +91,7 @@ unmarshal_model = function(model, inplace, ...) { #' @rdname marshaling #' @export -marshaled_model = function(model) { +is_marshaled_model = function(model) { test_class(model, "marshaled") } diff --git a/R/worker.R b/R/worker.R index 81fdb45fa..5da84ef5c 100644 --- a/R/worker.R +++ b/R/worker.R @@ -262,7 +262,7 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, learner = learner_train(learner, task, sets[["train"]], sets[["test"]], mode = mode, unmarshal = FALSE) model_marshaled = NULL - if (store_models && marshaled_model(learner$model) && !is_sequential) { + if (store_models && is_marshaled_model(learner$model) && !is_sequential) { if (is_sequential) { # callr + no parallelization learner$model = unmarshal_model(learner$model, clone = FALSE) diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index 2403e34df..7c254f524 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -414,10 +414,10 @@ expect_marshalable_learner = function(learner, task) { model = learner$model class_prev = class(model) expect_false(learner$marshaled) - expect_equal(marshaled_model(learner$model), learner$marshaled) + expect_equal(is_marshaled_model(learner$model), learner$marshaled) expect_invisible(learner$marshal()) expect_true(learner$marshaled) - expect_equal(marshaled_model(learner$model), learner$marshaled) + expect_equal(is_marshaled_model(learner$model), learner$marshaled) # cannot predict with marshaled learner expect_error(learner$predict(task), "has not been unmarshaled") diff --git a/man/marshaling.Rd b/man/marshaling.Rd index ea2eb504c..20526e85f 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -7,7 +7,7 @@ \alias{learner_marshaled} \alias{marshal_model} \alias{unmarshal_model} -\alias{marshaled_model} +\alias{is_marshaled_model} \title{(Un)marshal a Learner} \usage{ learner_unmarshal(.learner, ...) @@ -20,7 +20,7 @@ marshal_model(model, inplace, ...) unmarshal_model(model, inplace, ...) -marshaled_model(model) +is_marshaled_model(model) } \arguments{ \item{.learner}{\code{\link{Learner}}\cr @@ -45,7 +45,7 @@ root class should be set to \code{"marshaled"}. Which takes in the marshaled model and returns it in unmarshaled form. The generic takes care that the packages specified during \code{"marshal"} are loaded, and errs if they are not. The returned object must not inherit from class \code{"marshaled"}. -\item the function \code{marshaled_model(model)}, which returns \code{TRUE} if the model inherits from class \code{"marshaled"} +\item the function \verb{is_ marshaled_model(model)}, which returns \code{TRUE} if the model inherits from class \code{"marshaled"} and \code{FALSE} otherwise. } @@ -57,7 +57,7 @@ and the active binding \verb{$marshaled}. To make this as convenient as possible, the functions \code{learner_marshal(learner)}, \code{learner_unmarshal(learner)} and \code{learner_marshaled(learner)} are provided and can be called from within the public methods. All three functions throw an error if the learner is not trained and otherwise call -\code{marshal_model()}, \code{unmarshal_model()} or \code{marshaled_model()} on the learner's model. +\code{marshal_model()}, \code{unmarshal_model()} or \code{is_marshaled_model()} on the learner's model. You can verify whether you have correctly implemented marshaling by using the internal test helper \code{expect_marshalable_learner()}. This is also run by \code{expect_learner()} if a task is provided. From afbb6b541a6f6867946089e0981f2f0c944421c8 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 9 Apr 2024 13:39:09 +0200 Subject: [PATCH 27/47] ... --- NAMESPACE | 1 - R/Learner.R | 2 +- R/marshal.R | 53 ++++++++++++++++++++++------------- R/worker.R | 6 ++-- man/marshaling.Rd | 36 +++++++++++++++++++----- tests/testthat/test_marshal.R | 7 ++--- 6 files changed, 70 insertions(+), 35 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 8f3dfa130..8a3801296 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -88,7 +88,6 @@ S3method(head,Task) S3method(is_missing_prediction_data,PredictionDataClassif) S3method(is_missing_prediction_data,PredictionDataRegr) S3method(marshal_model,default) -S3method(marshal_model,marshaled) S3method(marshal_model,regr.debug_model) S3method(partition,Task) S3method(partition,TaskClassif) diff --git a/R/Learner.R b/R/Learner.R index 66b2dc90c..95e0b92c2 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -243,7 +243,7 @@ Learner = R6Class("Learner", test_row_ids = task$row_roles$test learner_train(learner, task, train_row_ids = train_row_ids, test_row_ids = test_row_ids, mode = mode) - self$model = unmarshal_model(self$state$model, clone = FALSE) + self$model = unmarshal_model(model = self$state$model, inplace = TRUE) # store data prototype proto = task$data(rows = integer()) diff --git a/R/marshal.R b/R/marshal.R index bda18830c..2587a59e3 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -3,26 +3,36 @@ #' @name marshaling #' #' @description -#' marshaling is the process of processing the model of a trained [`Learner`] so it an be successfully serialized and +#' Marshaling is the process of processing the model of a trained [`Learner`] so it an be successfully serialized and #' deserialized. The naming is inspired by the [marshal package](https://github.com/HenrikBengtsson/marshal) and we #' plan to fully migrate to this package once it is on CRAN. #' The supported implementation until then should therfore be considered as a temporary solution and is likely #' to change in the future. #' #' The central functions (and the only methods that are used by `mlr3` internally) are: -#' * the S3 generic `marshal_model(model, ...)`. +#' * the S3 generic `marshal_model(model, inplace, ...)`. #' Which takes in a model and returns it in marshaled form. #' The marshaled object should be a list with named elements `marshaled` and `packages`, where the former contains -#' the actual marshaled object, and the latter the packages required to unmarshal it. -#' This list should have as classes the classes of the original object with the suffix `"_marshaled"` added and the +#' the marshaled object, and the latter the packages required to unmarshal it. +#' This list should have the classes of the original object with the suffix `"_marshaled"` appended and the #' root class should be set to `"marshaled"`. -#' * the S3 generic `unmarshal_model(model, ...)`. +#' * the S3 generic `unmarshal_model(model, inplace ...)`. #' Which takes in the marshaled model and returns it in unmarshaled form. #' The generic takes care that the packages specified during `"marshal"` are loaded, and errs if they are not. #' The returned object must not inherit from class `"marshaled"`. #' * the function `is_ marshaled_model(model)`, which returns `TRUE` if the model inherits from class `"marshaled"` #' and `FALSE` otherwise. #' +#' +#' The contract of these functions is: +#' * `unmarshal_model(marshal_model(x))` returns `x` as is. +#' * If `is_marshaled_model(x)` is `TRUE`, this means that `x` is in marshaled form. +#' Note that it is not guarateed that `is_marshaled_model(marshal_model(x))` returns `TRUE`. +#' This is because the default `marshal_model(x)` returns `x` as-is. +#' +#' +#' @section Implementing Marshaling: +#' #' In order to implement marshaling for a Learner, you only need to overload the `marshal_model` and `unmarshal_model` #' methods and tag the learner with the `"marshal"` property accordingly. #' @@ -40,6 +50,15 @@ #' #' @param .learner [`Learner`]\cr #' The learner. +#' @param inplace (`logical(1)`)\cr +#' Whether to do the (un)marshaling in-place. +#' Only relevant for objects with reference semantics. +#' Set this to `TRUE` if you don't intend to keep the original object, e.g. in functions that return +#' a marshaled object. The default of methdos should always be `FALSE`, which should leave the original object +#' as-is. +#' @param ... (any)\cr +#' Additional parameters, currently unused. +#' #' @keywords internal #' @export learner_unmarshal = function(.learner, ...) { @@ -48,7 +67,7 @@ learner_unmarshal = function(.learner, ...) { stopf("Cannot unmarshal, Learner '%s' has not been trained yet", .learner$id) } # this will do nothing if the model was not marshaled - .learner$model = unmarshal_model(.learner$model, inplace = TRUE, ...) + .learner$model = unmarshal_model(model = .learner$model, inplace = TRUE, ...) invisible(.learner) } @@ -60,7 +79,7 @@ learner_marshal = function(.learner, ...) { stopf("Cannot marshal, Learner '%s' has not been trained yet", .learner$id) } # this will do nothing if the model was already marshaled - .learner$model = marshal_model(.learner$model, inplace = TRUE, ...) + .learner$model = marshal_model(model = .learner$model, inplace = TRUE, ...) invisible(.learner) } @@ -71,21 +90,23 @@ learner_marshaled = function(.learner) { if (is.null(.learner$model)) { stopf("Cannot check marshaled status, Learner '%s' has not been trained yet", .learner$id) } - is_marshaled_model(.learner$model) + is_marshaled_model(model = .learner$model) } #' @rdname marshaling #' @export -marshal_model = function(model, inplace, ...) { +marshal_model = function(model, inplace = FALSE, ...) { + assert_flag(inplace) UseMethod("marshal_model") } #' @rdname marshaling #' @export -unmarshal_model = function(model, inplace, ...) { +unmarshal_model = function(model, inplace = FALSE, ...) { if (is_marshaled_model(model) && is.character(model$packages)) { require_namespaces(model$packages) } + assert_flag(inplace) UseMethod("unmarshal_model") } @@ -97,22 +118,16 @@ is_marshaled_model = function(model) { #' @export marshal_model.default = function(model, ...) { - classes = class(model) - structure(list(marshaled = model), class = c(paste0(classes, "_marshaled"), "marshaled")) -} - -#' @export -marshal_model.marshaled = function(model, ...) { model } #' @export -unmarshal_model.marshaled = function(model, ...) { +unmarshal_model.marshaled = function(model, inplace = FALSE,...) { model[["marshaled"]] } #' @export -unmarshal_model.default = function(model, ...) { +unmarshal_model.default = function(model, inplace = FALSE, ...) { model } @@ -125,7 +140,7 @@ marshal_state_if_model = function(.state, ...) { unmarshal_state_if_model = function(.state, ...) { if (!is.null(.state$model)) { - .state$model = unmarshal_model(.state$model, ...) + .state$model = unmarshal_model(model = .state$model, ...) } .state } diff --git a/R/worker.R b/R/worker.R index 5da84ef5c..25f815049 100644 --- a/R/worker.R +++ b/R/worker.R @@ -265,11 +265,13 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, if (store_models && is_marshaled_model(learner$model) && !is_sequential) { if (is_sequential) { # callr + no parallelization - learner$model = unmarshal_model(learner$model, clone = FALSE) + learner$model = unmarshal_model(model = learner$model, inplace = TRUE) } else { # callr + parallelization + # we need to send the marshaled model back to the main process, so we temporarily keep a marshaled + # and unmarshalde model in RAM to avoig an additional call to marshal_mode model_marshaled = learner$model - learner$model = unmarshal_model(learner$model, clone = TRUE) + learner$model = unmarshal_model(model = learner$model, inplace = FALSE) } } diff --git a/man/marshaling.Rd b/man/marshaling.Rd index 20526e85f..03bd70345 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -16,18 +16,28 @@ learner_marshal(.learner, ...) learner_marshaled(.learner) -marshal_model(model, inplace, ...) +marshal_model(model, inplace = FALSE, ...) -unmarshal_model(model, inplace, ...) +unmarshal_model(model, inplace = FALSE, ...) is_marshaled_model(model) } \arguments{ \item{.learner}{\code{\link{Learner}}\cr The learner.} + +\item{...}{(any)\cr +Additional parameters, currently unused.} + +\item{inplace}{(\code{logical(1)})\cr +Whether to do the (un)marshaling in-place. +Only relevant for objects with reference semantics. +Set this to \code{TRUE} if you don't intend to keep the original object, e.g. in functions that return +a marshaled object. The default of methdos should always be \code{FALSE}, which should leave the original object +as-is.} } \description{ -marshaling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and +Marshaling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and deserialized. The naming is inspired by the \href{https://github.com/HenrikBengtsson/marshal}{marshal package} and we plan to fully migrate to this package once it is on CRAN. The supported implementation until then should therfore be considered as a temporary solution and is likely @@ -35,13 +45,13 @@ to change in the future. The central functions (and the only methods that are used by \code{mlr3} internally) are: \itemize{ -\item the S3 generic \code{marshal_model(model, ...)}. +\item the S3 generic \code{marshal_model(model, inplace, ...)}. Which takes in a model and returns it in marshaled form. The marshaled object should be a list with named elements \code{marshaled} and \code{packages}, where the former contains -the actual marshaled object, and the latter the packages required to unmarshal it. -This list should have as classes the classes of the original object with the suffix \code{"_marshaled"} added and the +the marshaled object, and the latter the packages required to unmarshal it. +This list should have the classes of the original object with the suffix \code{"_marshaled"} appended and the root class should be set to \code{"marshaled"}. -\item the S3 generic \code{unmarshal_model(model, ...)}. +\item the S3 generic \verb{unmarshal_model(model, inplace ...)}. Which takes in the marshaled model and returns it in unmarshaled form. The generic takes care that the packages specified during \code{"marshal"} are loaded, and errs if they are not. The returned object must not inherit from class \code{"marshaled"}. @@ -49,6 +59,17 @@ The returned object must not inherit from class \code{"marshaled"}. and \code{FALSE} otherwise. } +The contract of these functions is: +\itemize{ +\item \code{unmarshal_model(marshal_model(x))} returns \code{x} as is. +\item If \code{is_marshaled_model(x)} is \code{TRUE}, this means that \code{x} is in marshaled form. +Note that it is not guarateed that \code{is_marshaled_model(marshal_model(x))} returns \code{TRUE}. +This is because the default \code{marshal_model(x)} returns \code{x} as-is. +} +} +\section{Implementing Marshaling}{ + + In order to implement marshaling for a Learner, you only need to overload the \code{marshal_model} and \code{unmarshal_model} methods and tag the learner with the \code{"marshal"} property accordingly. @@ -64,4 +85,5 @@ You can verify whether you have correctly implemented marshaling by using the in For a concrete example on how to implement marshaling, see \code{\link{LearnerRegrDebug}}. } + \keyword{internal} diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R index 3224ac385..1aadedcc0 100644 --- a/tests/testthat/test_marshal.R +++ b/tests/testthat/test_marshal.R @@ -13,12 +13,9 @@ test_that("learner methods", { expect_false(learner_marshaled(learner)) }) -test_that("default method just changes class", { +test_that("default method does nothing", { x = 1 - xm = marshal_model(x) - expect_equal(class(xm), c("numeric_marshaled", "marshaled")) - expect_equal(xm[["marshaled"]], x) - expect_equal(x, unmarshal_model(xm)) + expect_equal(x, marshal_model(x)) }) test_that("marshaling a marshaled object does nothing", { From 26c6c8190898973bc1566abc9165a0914a565d9b Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 9 Apr 2024 15:09:33 +0200 Subject: [PATCH 28/47] marshal is property of classif.debug --- NAMESPACE | 6 +- R/LearnerClassifDebug.R | 87 ++++++++++++++++++++++------- R/LearnerRegrDebug.R | 46 +-------------- R/marshal.R | 2 +- inst/testthat/helper_expectations.R | 8 +++ man/marshaling.Rd | 2 +- man/mlr_learners_classif.debug.Rd | 48 ++++++++++++++++ man/mlr_learners_regr.debug.Rd | 48 ---------------- tests/testthat/test_HotstartStack.R | 4 +- tests/testthat/test_Learner.R | 4 +- tests/testthat/test_Measure.R | 10 ++-- tests/testthat/test_benchmark.R | 18 +++--- tests/testthat/test_marshal.R | 18 +++--- tests/testthat/test_mlr_learners.R | 4 +- tests/testthat/test_resample.R | 18 +++--- 15 files changed, 167 insertions(+), 156 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 8a3801296..2d4297976 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -87,8 +87,8 @@ S3method(fix_factor_levels,data.table) S3method(head,Task) S3method(is_missing_prediction_data,PredictionDataClassif) S3method(is_missing_prediction_data,PredictionDataRegr) +S3method(marshal_model,classif.debug_model) S3method(marshal_model,default) -S3method(marshal_model,regr.debug_model) S3method(partition,Task) S3method(partition,TaskClassif) S3method(partition,TaskRegr) @@ -106,9 +106,9 @@ S3method(set_threads,default) S3method(set_threads,list) S3method(summary,Task) S3method(tail,Task) +S3method(unmarshal_model,classif.debug_model_marshaled) S3method(unmarshal_model,default) S3method(unmarshal_model,marshaled) -S3method(unmarshal_model,regr.debug_model_marshaled) export(BenchmarkResult) export(DataBackend) export(DataBackendDataTable) @@ -247,8 +247,6 @@ importFrom(R6,R6Class) importFrom(R6,is.R6) importFrom(RhpcBLASctl,blas_get_num_procs) importFrom(RhpcBLASctl,blas_set_num_threads) -importFrom(data.table,as.data.table) -importFrom(data.table,data.table) importFrom(future,nbrOfWorkers) importFrom(future,plan) importFrom(graphics,plot) diff --git a/R/LearnerClassifDebug.R b/R/LearnerClassifDebug.R index 8abd78b5a..195376925 100644 --- a/R/LearnerClassifDebug.R +++ b/R/LearnerClassifDebug.R @@ -24,6 +24,7 @@ #' \item{warning_train:}{Probability to signal a warning during train.} #' \item{x:}{Numeric tuning parameter. Has no effect.} #' \item{iter:}{Integer parameter for testing hotstarting.} +#' \item{count_marshaling:}{If `TRUE`, `marshal_model` will increase the `marshal_count` by 1. This value is initialized to `FALSE`.} #' } #' Note that segfaults may not be triggered reliably on your operating system. #' Also note that if they work as intended, they will tear down your R session immediately! @@ -49,39 +50,62 @@ LearnerClassifDebug = R6Class("LearnerClassifDebug", inherit = LearnerClassif, #' @description #' Creates a new instance of this [R6][R6::R6Class] class. initialize = function() { + param_set = ps( + error_predict = p_dbl(0, 1, default = 0, tags = "predict"), + error_train = p_dbl(0, 1, default = 0, tags = "train"), + message_predict = p_dbl(0, 1, default = 0, tags = "predict"), + message_train = p_dbl(0, 1, default = 0, tags = "train"), + predict_missing = p_dbl(0, 1, default = 0, tags = "predict"), + predict_missing_type = p_fct(c("na", "omit"), default = "na", tags = "predict"), + save_tasks = p_lgl(default = FALSE, tags = c("train", "predict")), + segfault_predict = p_dbl(0, 1, default = 0, tags = "predict"), + segfault_train = p_dbl(0, 1, default = 0, tags = "train"), + sleep_train = p_uty(tags = "train"), + sleep_predict = p_uty(tags = "predict"), + threads = p_int(1L, tags = c("train", "threads")), + warning_predict = p_dbl(0, 1, default = 0, tags = "predict"), + warning_train = p_dbl(0, 1, default = 0, tags = "train"), + x = p_dbl(0, 1, tags = "train"), + iter = p_int(1, default = 1, tags = c("train", "hotstart")), + count_marshaling = p_lgl(default = FALSE, tags = "train") + ) super$initialize( id = "classif.debug", + param_set = param_set, feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), predict_types = c("response", "prob"), - param_set = ps( - error_predict = p_dbl(0, 1, default = 0, tags = "predict"), - error_train = p_dbl(0, 1, default = 0, tags = "train"), - message_predict = p_dbl(0, 1, default = 0, tags = "predict"), - message_train = p_dbl(0, 1, default = 0, tags = "train"), - predict_missing = p_dbl(0, 1, default = 0, tags = "predict"), - predict_missing_type = p_fct(c("na", "omit"), default = "na", tags = "predict"), - save_tasks = p_lgl(default = FALSE, tags = c("train", "predict")), - segfault_predict = p_dbl(0, 1, default = 0, tags = "predict"), - segfault_train = p_dbl(0, 1, default = 0, tags = "train"), - sleep_train = p_uty(tags = "train"), - sleep_predict = p_uty(tags = "predict"), - threads = p_int(1L, tags = c("train", "threads")), - warning_predict = p_dbl(0, 1, default = 0, tags = "predict"), - warning_train = p_dbl(0, 1, default = 0, tags = "train"), - x = p_dbl(0, 1, tags = "train"), - iter = p_int(1, default = 1, tags = c("train", "hotstart")) - ), - properties = c("twoclass", "multiclass", "missings", "hotstart_forward"), + properties = c("twoclass", "multiclass", "missings", "hotstart_forward", "marshal"), man = "mlr3::mlr_learners_classif.debug", data_formats = c("data.table", "Matrix"), label = "Debug Learner for Classification" ) + }, + #' @description + #' Marshals the learner. + #' @param ... (any)\cr + #' Additional arguments passed to [`marshal_model()`]. + marshal = function(...) { + learner_marshal(.learner = self, ...) + }, + #' @description + #' Unmarshal the learner. + #' @param ... (any)\cr + #' Additional arguments passed to [`unmarshal_model()`]. + unmarshal = function(...) { + learner_unmarshal(.learner = self, ...) + } + ), + active = list( + #' @field marshaled (logical(1))\cr + #' Whether the learner has been marshaled. + marshaled = function() { + learner_marshaled(self) } ), - private = list( .train = function(task) { pv = self$param_set$get_values(tags = "train") + pv$count_marshaling = pv$count_marshaling %??% FALSE roll = function(name) { name %in% names(pv) && pv[[name]] > runif(1L) } @@ -110,6 +134,10 @@ LearnerClassifDebug = R6Class("LearnerClassifDebug", inherit = LearnerClassif, model$task_train = task$clone(deep = TRUE) } + if (isTRUE(pv$count_marshaling)) { + model$marshal_count = 0L + } + set_class(model, "classif.debug_model") }, @@ -193,3 +221,22 @@ LearnerClassifDebug = R6Class("LearnerClassifDebug", inherit = LearnerClassif, #' @include mlr_learners.R mlr_learners$add("classif.debug", function() LearnerClassifDebug$new()) + + + + +#' @export +#' @method marshal_model classif.debug_model +marshal_model.classif.debug_model = function(model, ...) { + if (!is.null(model$marshal_count)) { + model$marshal_count = model$marshal_count + 1 + } + newclass = c("regr.debug_model_marshaled", "marshaled") + structure(list(marshaled = model, packages = "mlr3"), class = newclass) +} + +#' @export +#' @method unmarshal_model classif.debug_model_marshaled +unmarshal_model.classif.debug_model_marshaled = function(model, ...) { + model$marshaled +} diff --git a/R/LearnerRegrDebug.R b/R/LearnerRegrDebug.R index 11551da73..f8be7dc49 100644 --- a/R/LearnerRegrDebug.R +++ b/R/LearnerRegrDebug.R @@ -13,7 +13,6 @@ #' \item{save_tasks:}{Saves input task in `model` slot during training and prediction.} #' \item{threads:}{Number of threads to use. Has no effect.} #' \item{x:}{Numeric tuning parameter. Has no effect.} -#' \item{count_marshaling:}{If `TRUE`, `marshal_model` will increase the `marshal_count` by 1. This value is initialized to `FALSE`.} #' } #' #' @templateVar id regr.debug @@ -39,10 +38,8 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, predict_missing_type = p_fct(c("na", "omit"), default = "na", tags = "predict"), save_tasks = p_lgl(default = FALSE, tags = c("train", "predict")), threads = p_int(1L, tags = c("train", "threads")), - x = p_dbl(0, 1, tags = "train"), - count_marshaling = p_lgl(tags = c("train", "required")) + x = p_dbl(0, 1, tags = "train") ) - param_set$set_values(count_marshaling = FALSE) super$initialize( id = "regr.debug", feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), @@ -53,27 +50,6 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, data_formats = c("data.table", "Matrix"), label = "Debug Learner for Regression" ) - }, - #' @description - #' Marshals the learner. - #' @param ... (any)\cr - #' Additional arguments passed to [`marshal_model()`]. - marshal = function(...) { - learner_marshal(.learner = self, ...) - }, - #' @description - #' Unmarshal the learner. - #' @param ... (any)\cr - #' Additional arguments passed to [`unmarshal_model()`]. - unmarshal = function(...) { - learner_unmarshal(.learner = self, ...) - } - ), - active = list( - #' @field marshaled (logical(1))\cr - #' Whether the learner has been marshaled. - marshaled = function() { - learner_marshaled(self) } ), private = list( @@ -89,9 +65,6 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, if (isTRUE(pv$save_tasks)) { model$task_train = task$clone(deep = TRUE) } - if (isTRUE(pv$count_marshaling)) { - model$marshal_count = 0L - } set_class(model, "regr.debug_model") }, @@ -127,20 +100,3 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, #' @include mlr_learners.R mlr_learners$add("regr.debug", function() LearnerRegrDebug$new()) - - -#' @export -#' @method marshal_model regr.debug_model -marshal_model.regr.debug_model = function(model, ...) { - if (!is.null(model$marshal_count)) { - model$marshal_count = model$marshal_count + 1 - } - newclass = c("regr.debug_model_marshaled", "marshaled") - structure(list(marshaled = model, packages = "mlr3"), class = newclass) -} - -#' @export -#' @method unmarshal_model regr.debug_model_marshaled -unmarshal_model.regr.debug_model_marshaled = function(model, ...) { - model$marshaled -} diff --git a/R/marshal.R b/R/marshal.R index 2587a59e3..475d8d2aa 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -46,7 +46,7 @@ #' You can verify whether you have correctly implemented marshaling by using the internal test helper #' `expect_marshalable_learner()`. This is also run by `expect_learner()` if a task is provided. #' -#' For a concrete example on how to implement marshaling, see [`LearnerRegrDebug`]. +#' For a concrete example on how to implement marshaling, see [`LearnerClassifDebug`]. #' #' @param .learner [`Learner`]\cr #' The learner. diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index 7c254f524..d2bb7c3fd 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -375,6 +375,14 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { info = sprintf("All hyperparameters of learner %s must be tagged with 'train' or 'predict'. Missing tags for: %s", lrn$id, paste0(names(tags), collapse = ", ")) ) + if ("marshal" %in% lrn$properties) { + assert_function(lrn$marshal) + assert_function(lrn$unmarshal) + } else if (exists("marshal", lrn)) { + assert_function(lrn$unmarshal) + assert_true("marshal" %in% lrn$properties) + } + if (!is.null(task)) { checkmate::expect_class(task, "Task") checkmate::expect_subset(lrn$properties, mlr3::mlr_reflections$learner_properties[[task$task_type]]) diff --git a/man/marshaling.Rd b/man/marshaling.Rd index 03bd70345..036373459 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -83,7 +83,7 @@ All three functions throw an error if the learner is not trained and otherwise c You can verify whether you have correctly implemented marshaling by using the internal test helper \code{expect_marshalable_learner()}. This is also run by \code{expect_learner()} if a task is provided. -For a concrete example on how to implement marshaling, see \code{\link{LearnerRegrDebug}}. +For a concrete example on how to implement marshaling, see \code{\link{LearnerClassifDebug}}. } \keyword{internal} diff --git a/man/mlr_learners_classif.debug.Rd b/man/mlr_learners_classif.debug.Rd index bef639870..9c1d9b404 100644 --- a/man/mlr_learners_classif.debug.Rd +++ b/man/mlr_learners_classif.debug.Rd @@ -25,6 +25,7 @@ The following hyperparameters trigger the following actions: \item{warning_train:}{Probability to signal a warning during train.} \item{x:}{Numeric tuning parameter. Has no effect.} \item{iter:}{Integer parameter for testing hotstarting.} +\item{count_marshaling:}{If \code{TRUE}, \code{marshal_model} will increase the \code{marshal_count} by 1. This value is initialized to \code{FALSE}.} } Note that segfaults may not be triggered reliably on your operating system. Also note that if they work as intended, they will tear down your R session immediately! @@ -67,6 +68,7 @@ lrn("classif.debug") warning_train \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr x \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr iter \tab integer \tab 1 \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr + count_marshaling \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr } } @@ -116,10 +118,20 @@ Other Learner: \section{Super classes}{ \code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerClassif]{mlr3::LearnerClassif}} -> \code{LearnerClassifDebug} } +\section{Active bindings}{ +\if{html}{\out{
}} +\describe{ +\item{\code{marshaled}}{(logical(1))\cr +Whether the learner has been marshaled.} +} +\if{html}{\out{
}} +} \section{Methods}{ \subsection{Public methods}{ \itemize{ \item \href{#method-LearnerClassifDebug-new}{\code{LearnerClassifDebug$new()}} +\item \href{#method-LearnerClassifDebug-marshal}{\code{LearnerClassifDebug$marshal()}} +\item \href{#method-LearnerClassifDebug-unmarshal}{\code{LearnerClassifDebug$unmarshal()}} \item \href{#method-LearnerClassifDebug-clone}{\code{LearnerClassifDebug$clone()}} } } @@ -146,6 +158,42 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. \if{html}{\out{
}}\preformatted{LearnerClassifDebug$new()}\if{html}{\out{
}} } +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerClassifDebug-marshal}{}}} +\subsection{Method \code{marshal()}}{ +Marshals the learner. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerClassifDebug$marshal(...)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=marshal_model]{marshal_model()}}.} +} +\if{html}{\out{
}} +} +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-LearnerClassifDebug-unmarshal}{}}} +\subsection{Method \code{unmarshal()}}{ +Unmarshal the learner. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{LearnerClassifDebug$unmarshal(...)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{...}}{(any)\cr +Additional arguments passed to \code{\link[=unmarshal_model]{unmarshal_model()}}.} +} +\if{html}{\out{
}} +} } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/mlr_learners_regr.debug.Rd b/man/mlr_learners_regr.debug.Rd index d6c186457..2c2c53b23 100644 --- a/man/mlr_learners_regr.debug.Rd +++ b/man/mlr_learners_regr.debug.Rd @@ -14,7 +14,6 @@ The following hyperparameters trigger the following actions: \item{save_tasks:}{Saves input task in \code{model} slot during training and prediction.} \item{threads:}{Number of threads to use. Has no effect.} \item{x:}{Numeric tuning parameter. Has no effect.} -\item{count_marshaling:}{If \code{TRUE}, \code{marshal_model} will increase the \code{marshal_count} by 1. This value is initialized to \code{FALSE}.} } } \section{Dictionary}{ @@ -44,7 +43,6 @@ lrn("regr.debug") save_tasks \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr threads \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr x \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - count_marshaling \tab logical \tab - \tab TRUE, FALSE \tab - \cr } } @@ -91,20 +89,10 @@ Other Learner: \section{Super classes}{ \code{\link[mlr3:Learner]{mlr3::Learner}} -> \code{\link[mlr3:LearnerRegr]{mlr3::LearnerRegr}} -> \code{LearnerRegrDebug} } -\section{Active bindings}{ -\if{html}{\out{
}} -\describe{ -\item{\code{marshaled}}{(logical(1))\cr -Whether the learner has been marshaled.} -} -\if{html}{\out{
}} -} \section{Methods}{ \subsection{Public methods}{ \itemize{ \item \href{#method-LearnerRegrDebug-new}{\code{LearnerRegrDebug$new()}} -\item \href{#method-LearnerRegrDebug-marshal}{\code{LearnerRegrDebug$marshal()}} -\item \href{#method-LearnerRegrDebug-unmarshal}{\code{LearnerRegrDebug$unmarshal()}} \item \href{#method-LearnerRegrDebug-clone}{\code{LearnerRegrDebug$clone()}} } } @@ -131,42 +119,6 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. \if{html}{\out{
}}\preformatted{LearnerRegrDebug$new()}\if{html}{\out{
}} } -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrDebug-marshal}{}}} -\subsection{Method \code{marshal()}}{ -Marshals the learner. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrDebug$marshal(...)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{...}}{(any)\cr -Additional arguments passed to \code{\link[=marshal_model]{marshal_model()}}.} -} -\if{html}{\out{
}} -} -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-LearnerRegrDebug-unmarshal}{}}} -\subsection{Method \code{unmarshal()}}{ -Unmarshal the learner. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{LearnerRegrDebug$unmarshal(...)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{...}}{(any)\cr -Additional arguments passed to \code{\link[=unmarshal_model]{unmarshal_model()}}.} -} -\if{html}{\out{
}} -} } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/tests/testthat/test_HotstartStack.R b/tests/testthat/test_HotstartStack.R index 7d6766f7c..55148c8c3 100644 --- a/tests/testthat/test_HotstartStack.R +++ b/tests/testthat/test_HotstartStack.R @@ -412,8 +412,8 @@ test_that("HotstartStack threshold works", { test_that("error when adding marshaled learner", { hot = HotstartStack$new() - learner = lrn("regr.debug") - learner$train(tsk("mtcars")) + learner = lrn("classif.debug") + learner$train(tsk("iris")) learner$marshal() expect_error(hot$add(learner), "unmarshaled") }) diff --git a/tests/testthat/test_Learner.R b/tests/testthat/test_Learner.R index 519788976..910fafde0 100644 --- a/tests/testthat/test_Learner.R +++ b/tests/testthat/test_Learner.R @@ -326,8 +326,8 @@ test_that("Models can be replaced", { }) test_that("marshaling and encapsulation", { - task = tsk("mtcars") - learner = lrn("regr.debug", count_marshaling = TRUE) + task = tsk("iris") + learner = lrn("classif.debug", count_marshaling = TRUE) # callr encapsulation causes marshaling learner$encapsulate = c(train = "callr") diff --git a/tests/testthat/test_Measure.R b/tests/testthat/test_Measure.R index a29a08639..6b2ff1de1 100644 --- a/tests/testthat/test_Measure.R +++ b/tests/testthat/test_Measure.R @@ -135,8 +135,10 @@ test_that("scoring fails when measure requires_model, but model is in marshaled measure = msr("classif.acc") measure$properties = c(measure$properties, "requires_model") - rr = resample(tsk("mtcars"), lrn("regr.debug"), rsmp("holdout")) - rr$marshal() - - rr$score(msr("selected_features")) + task = tsk("iris") + learner = lrn("classif.debug") + pred = learner$train(task)$predict(task) + learner$marshal() + expect_error(measure$score(pred, learner = learner), + regexp = "is in marshaled form") }) diff --git a/tests/testthat/test_benchmark.R b/tests/testthat/test_benchmark.R index bf55952c1..eafc08ba0 100644 --- a/tests/testthat/test_benchmark.R +++ b/tests/testthat/test_benchmark.R @@ -479,8 +479,8 @@ test_that("param_values in benchmark", { test_that("parallel execution automatically triggers marshaling", { - learner = lrn("regr.debug", count_marshaling = TRUE) - task = tsk("mtcars") + learner = lrn("classif.debug", count_marshaling = TRUE) + task = tsk("iris") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) bmr = with_future(future::multisession, { @@ -491,8 +491,8 @@ test_that("parallel execution automatically triggers marshaling", { }) test_that("sequential execution does not trigger marshaling", { - learner = lrn("regr.debug", count_marshaling = TRUE) - task = tsk("mtcars") + learner = lrn("classif.debug", count_marshaling = TRUE) + task = tsk("iris") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) bmr = with_future(future::sequential, { @@ -502,8 +502,8 @@ test_that("sequential execution does not trigger marshaling", { }) test_that("parallel execution and callr marshal once", { - learner = lrn("regr.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) - task = tsk("mtcars") + learner = lrn("classif.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + task = tsk("iris") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) bmr = with_future(future::multisession, { @@ -515,8 +515,8 @@ test_that("parallel execution and callr marshal once", { test_that("unmarshal parameter is respected", { - learner = lrn("regr.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) - task = tsk("mtcars") + learner = lrn("classif.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + task = tsk("iris") resampling = rsmp("holdout") design = benchmark_grid(task, learner, resampling) bmr = with_future(future::multisession, { @@ -530,7 +530,7 @@ test_that("unmarshal parameter is respected", { }) test_that("BenchmarkResult can be (un)marshaled", { - bmr = benchmark(benchmark_grid(tsk("mtcars"), lrn("regr.debug"), rsmp("holdout")), store_models = TRUE) + bmr = benchmark(benchmark_grid(tsk("iris"), lrn("classif.debug"), rsmp("holdout")), store_models = TRUE) expect_false(bmr$resample_result(1)$learners[[1]]$marshaled) bmr$marshal() expect_true(bmr$resample_result(1)$learners[[1]]$marshaled) diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R index 1aadedcc0..8cea6fccc 100644 --- a/tests/testthat/test_marshal.R +++ b/tests/testthat/test_marshal.R @@ -1,6 +1,6 @@ test_that("learner methods", { - learner = lrn("regr.debug") - task = tsk("mtcars") + learner = lrn("classif.debug") + task = tsk("iris") expect_error(learner_marshal(learner), "not been trained") expect_error(learner_unmarshal(learner), "not been trained") expect_error(learner_marshaled(learner), "not been trained") @@ -32,9 +32,9 @@ test_that("unmarshaling a unmarshaled object does nothing", { }) -test_that("marshaling for LearnerRegrDebug", { - task = tsk("mtcars") - learner = lrn("regr.debug") +test_that("marshaling for LearnerClassif", { + task = tsk("iris") + learner = lrn("classif.debug") learner$train(task) expect_false(learner$marshaled) learner$marshal() @@ -44,12 +44,12 @@ test_that("marshaling for LearnerRegrDebug", { expect_learner(learner, task) }) -test_that("marshal count works for LearnerRegrDebug", { +test_that("marshal count works for LearnerClassifDebug", { # to mock that marshaling behaves as expected, we need to be know how often it happened # note that this means that marshal_model modifies the model in a permanent way, i.e. this is not reversed by # unmarshal_model. - learner = lrn("regr.debug", count_marshaling = TRUE) - task = tsk("mtcars") + learner = lrn("classif.debug", count_marshaling = TRUE) + task = tsk("iris") learner$train(task) expect_equal(learner$model$marshal_count, 0) learner$marshal()$unmarshal() @@ -59,7 +59,7 @@ test_that("marshal count works for LearnerRegrDebug", { # TO make the lily learner more realistic (i.e. (un)marshaling leaves the object unchanged) # the count_marshaling parameter can also be set to FALSE - learner2 = lrn("regr.debug", count_marshaling = FALSE) + learner2 = lrn("classif.debug", count_marshaling = FALSE) learner2$train(task) expect_true(is.null(learner2$model$marshal_count)) model1 = learner2$model diff --git a/tests/testthat/test_mlr_learners.R b/tests/testthat/test_mlr_learners.R index 9ab2ab74b..ded6ad7ac 100644 --- a/tests/testthat/test_mlr_learners.R +++ b/tests/testthat/test_mlr_learners.R @@ -4,8 +4,8 @@ test_that("mlr_learners", { for (key in keys) { l = lrn(key) - if (key == "regr.debug") { - expect_learner(l, task = tsk("mtcars")) + if (key == "classif.debug") { + expect_learner(l, task = tsk("iris")) } else { expect_learner(l) } diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index 521585164..f2f42cd6b 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -158,8 +158,8 @@ test_that("as_resample_result works for result data", { }) test_that("parallel execution automatically triggers marshaling", { - learner = lrn("regr.debug", count_marshaling = TRUE) - task = tsk("mtcars") + learner = lrn("classif.debug", count_marshaling = TRUE) + task = tsk("iris") resampling = rsmp("holdout") rr = with_future(future::multisession, { resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) @@ -169,8 +169,8 @@ test_that("parallel execution automatically triggers marshaling", { }) test_that("sequential execution does not trigger marshaling", { - learner = lrn("regr.debug", count_marshaling = TRUE) - task = tsk("mtcars") + learner = lrn("classif.debug", count_marshaling = TRUE) + task = tsk("iris") resampling = rsmp("holdout") rr = with_future(future::sequential, { resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) @@ -179,8 +179,8 @@ test_that("sequential execution does not trigger marshaling", { }) test_that("parallel execution and callr marshal once", { - learner = lrn("regr.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) - task = tsk("mtcars") + learner = lrn("classif.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + task = tsk("iris") resampling = rsmp("holdout") rr = with_future(future::multisession, { resample(task, learner, resampling, store_models = TRUE, unmarshal = TRUE) @@ -191,8 +191,8 @@ test_that("parallel execution and callr marshal once", { test_that("unmarshal parameter is respected", { - learner = lrn("regr.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) - task = tsk("mtcars") + learner = lrn("classif.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + task = tsk("iris") resampling = rsmp("holdout") rr = with_future(future::multisession, { list( @@ -205,7 +205,7 @@ test_that("unmarshal parameter is respected", { }) test_that("ResampleResult can be (un)marshaled", { - rr = resample(tsk("mtcars"), lrn("regr.debug"), rsmp("holdout"), store_models = TRUE) + rr = resample(tsk("iris"), lrn("classif.debug"), rsmp("holdout"), store_models = TRUE) expect_false(rr$learners[[1]]$marshaled) rr$marshal() expect_true(rr$learners[[1]]$marshaled) From 2f5e6858e63284ed19b8a2f357482b6d60075e86 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 9 Apr 2024 17:03:34 +0200 Subject: [PATCH 29/47] fix printer and autotest --- NAMESPACE | 3 +++ R/marshal.R | 5 +++++ inst/testthat/helper_expectations.R | 3 --- tests/testthat/test_marshal.R | 4 ++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 2d4297976..7464c9018 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -97,6 +97,7 @@ S3method(print,PredictionData) S3method(print,benchmark_grid) S3method(print,bmr_aggregate) S3method(print,bmr_score) +S3method(print,marshaled) S3method(print,rr_score) S3method(rd_info,Learner) S3method(rd_info,Measure) @@ -247,6 +248,8 @@ importFrom(R6,R6Class) importFrom(R6,is.R6) importFrom(RhpcBLASctl,blas_get_num_procs) importFrom(RhpcBLASctl,blas_set_num_threads) +importFrom(data.table,as.data.table) +importFrom(data.table,data.table) importFrom(future,nbrOfWorkers) importFrom(future,plan) importFrom(graphics,plot) diff --git a/R/marshal.R b/R/marshal.R index 475d8d2aa..de177a015 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -144,3 +144,8 @@ unmarshal_state_if_model = function(.state, ...) { } .state } + +#' @export +print.marshaled = function(x, ...) { + catn(sprintf("<%s>", class(x)[[1L]])) +} diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index d2bb7c3fd..8fa7876d5 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -378,9 +378,6 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { if ("marshal" %in% lrn$properties) { assert_function(lrn$marshal) assert_function(lrn$unmarshal) - } else if (exists("marshal", lrn)) { - assert_function(lrn$unmarshal) - assert_true("marshal" %in% lrn$properties) } if (!is.null(task)) { diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R index 8cea6fccc..086d0fe6f 100644 --- a/tests/testthat/test_marshal.R +++ b/tests/testthat/test_marshal.R @@ -66,3 +66,7 @@ test_that("marshal count works for LearnerClassifDebug", { model2 = learner2$marshal()$unmarshal()$model expect_equal(model1, model2) }) + +test_that("printer", { + expect_output(print(lrn("classif.debug")$train(tsk("iris"))$marshal()$model), "") +}) From 51e5c5fa0910abdbd69f76f0ac4163d0f9a733fb Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 9 Apr 2024 18:38:21 +0200 Subject: [PATCH 30/47] ... --- inst/testthat/helper_expectations.R | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index d4fca1c53..5465cc152 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -388,8 +388,6 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { if ("marshal" %in% lrn$properties) { expect_marshalable_learner(lrn, task) } - } else if ("marshal" %in% lrn$properties) { - message("Cannot test 'marshal' property of the learner as no task is provided.") } if (!inherits(lrn, "GraphLearner") && !inherits(lrn, "AutoTuner")) { # still not in pipelines, breaking check in mlr3tuning @@ -421,11 +419,15 @@ expect_marshalable_learner = function(learner, task) { expect_false(learner$marshaled) expect_equal(is_marshaled_model(learner$model), learner$marshaled) expect_invisible(learner$marshal()) - expect_true(learner$marshaled) + if (!test_class(learner, "GraphLearner")) { + expect_true(learner$marshaled) + } expect_equal(is_marshaled_model(learner$model), learner$marshaled) - # cannot predict with marshaled learner - expect_error(learner$predict(task), "has not been unmarshaled") + if (learner$marshaled) { + # cannot predict with marshaled learner + expect_error(learner$predict(task), "has not been unmarshaled") + } # unmarshaling works expect_invisible(learner$unmarshal()) @@ -442,8 +444,6 @@ expect_marshalable_learner = function(learner, task) { learner$predict(task) expect_false(learner$train(task)$marshaled) - - } expect_resampling = function(r, task = NULL) { From ad8cfb5c630093dc0ad724a3d5eadd9c2acb9497 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 9 Apr 2024 18:50:04 +0200 Subject: [PATCH 31/47] typo --- tests/testthat/test_resample.R | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index fedff0e72..b5b53361d 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -217,6 +217,7 @@ test_that("ResampleResult can be (un)marshaled", { model = rr1$learners[[1]]$model rr1$unmarshal() expect_equal(rr1$learners[[1]]$model, model) +}) test_that("does not unnecessarily clone state", { task = tsk("iris") From 9581425f6a824a0d81b3144cda4efe77cbac16d8 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 9 Apr 2024 19:01:05 +0200 Subject: [PATCH 32/47] docs --- R/marshal.R | 15 ++++++--------- man/marshaling.Rd | 14 ++++++-------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/R/marshal.R b/R/marshal.R index de177a015..c1cf93369 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -23,13 +23,16 @@ #' * the function `is_ marshaled_model(model)`, which returns `TRUE` if the model inherits from class `"marshaled"` #' and `FALSE` otherwise. #' -#' -#' The contract of these functions is: +#' The contract of these generics is: #' * `unmarshal_model(marshal_model(x))` returns `x` as is. +#' * `unmarshal_model(x)` can be serialized and de-serialzed without loss of information. #' * If `is_marshaled_model(x)` is `TRUE`, this means that `x` is in marshaled form. #' Note that it is not guarateed that `is_marshaled_model(marshal_model(x))` returns `TRUE`. #' This is because the default `marshal_model(x)` returns `x` as-is. -#' +#' * The `inplace` argument determines whether in-place marshaling should be performed. +#' This is especially relevant in the context of references semantics. +#' If `inplace` is `FALSE`, the marshaled model should not share references with the original model. +#' If `inplace` is `TRUE` this is not necessary and the original object can be modified in-place. #' #' @section Implementing Marshaling: #' @@ -50,12 +53,6 @@ #' #' @param .learner [`Learner`]\cr #' The learner. -#' @param inplace (`logical(1)`)\cr -#' Whether to do the (un)marshaling in-place. -#' Only relevant for objects with reference semantics. -#' Set this to `TRUE` if you don't intend to keep the original object, e.g. in functions that return -#' a marshaled object. The default of methdos should always be `FALSE`, which should leave the original object -#' as-is. #' @param ... (any)\cr #' Additional parameters, currently unused. #' diff --git a/man/marshaling.Rd b/man/marshaling.Rd index 036373459..fc32d024c 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -28,13 +28,6 @@ The learner.} \item{...}{(any)\cr Additional parameters, currently unused.} - -\item{inplace}{(\code{logical(1)})\cr -Whether to do the (un)marshaling in-place. -Only relevant for objects with reference semantics. -Set this to \code{TRUE} if you don't intend to keep the original object, e.g. in functions that return -a marshaled object. The default of methdos should always be \code{FALSE}, which should leave the original object -as-is.} } \description{ Marshaling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and @@ -59,12 +52,17 @@ The returned object must not inherit from class \code{"marshaled"}. and \code{FALSE} otherwise. } -The contract of these functions is: +The contract of these generics is: \itemize{ \item \code{unmarshal_model(marshal_model(x))} returns \code{x} as is. +\item \code{unmarshal_model(x)} can be serialized and de-serialzed without loss of information. \item If \code{is_marshaled_model(x)} is \code{TRUE}, this means that \code{x} is in marshaled form. Note that it is not guarateed that \code{is_marshaled_model(marshal_model(x))} returns \code{TRUE}. This is because the default \code{marshal_model(x)} returns \code{x} as-is. +\item The \code{inplace} argument determines whether in-place marshaling should be performed. +This is especially relevant in the context of references semantics. +If \code{inplace} is \code{FALSE}, the marshaled model should not share references with the original model. +If \code{inplace} is \code{TRUE} this is not necessary and the original object can be modified in-place. } } \section{Implementing Marshaling}{ From 72733a9cf19e93edf29ab82928bc21218e382dc2 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 10 Apr 2024 16:57:07 +0200 Subject: [PATCH 33/47] refactor --- R/LearnerClassifDebug.R | 3 -- R/marshal.R | 4 +-- R/worker.R | 58 +++++++++++++++++++++++----------- man/marshaling.Rd | 4 +-- tests/testthat/test_resample.R | 11 +++++++ 5 files changed, 54 insertions(+), 26 deletions(-) diff --git a/R/LearnerClassifDebug.R b/R/LearnerClassifDebug.R index 195376925..3f41d5067 100644 --- a/R/LearnerClassifDebug.R +++ b/R/LearnerClassifDebug.R @@ -222,9 +222,6 @@ LearnerClassifDebug = R6Class("LearnerClassifDebug", inherit = LearnerClassif, #' @include mlr_learners.R mlr_learners$add("classif.debug", function() LearnerClassifDebug$new()) - - - #' @export #' @method marshal_model classif.debug_model marshal_model.classif.debug_model = function(model, ...) { diff --git a/R/marshal.R b/R/marshal.R index c1cf93369..cc2e9a945 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -31,8 +31,8 @@ #' This is because the default `marshal_model(x)` returns `x` as-is. #' * The `inplace` argument determines whether in-place marshaling should be performed. #' This is especially relevant in the context of references semantics. -#' If `inplace` is `FALSE`, the marshaled model should not share references with the original model. -#' If `inplace` is `TRUE` this is not necessary and the original object can be modified in-place. +#' If `inplace` is `FALSE`, the original input should not be modified, otherwise this is allowed. +#' Note that the input and output can still share references, even when `inplace` is `FALSE`. #' #' @section Implementing Marshaling: #' diff --git a/R/worker.R b/R/worker.R index 25f815049..9121013c2 100644 --- a/R/worker.R +++ b/R/worker.R @@ -261,19 +261,15 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, learner_hash = learner$hash learner = learner_train(learner, task, sets[["train"]], sets[["test"]], mode = mode, unmarshal = FALSE) - model_marshaled = NULL - if (store_models && is_marshaled_model(learner$model) && !is_sequential) { - if (is_sequential) { - # callr + no parallelization - learner$model = unmarshal_model(model = learner$model, inplace = TRUE) - } else { - # callr + parallelization - # we need to send the marshaled model back to the main process, so we temporarily keep a marshaled - # and unmarshalde model in RAM to avoig an additional call to marshal_mode - model_marshaled = learner$model - learner$model = unmarshal_model(model = learner$model, inplace = FALSE) - } - } + # The model is in marshaled form in case we did callr encapsulation and the model needed marshaling. + # But for the predict step we need it unmarshaled. + # In case we have a marshaled model and are parallelizing, we avoid to marshal again after unmarshaling + # the model for the predict step. + # We do this by temporarily keep the marshaled and unmarshaled model in RAM (the marshaled model model of course only + # if store_models is TRUE) + model_marshaled_or_null = unmarshal_maybe_after_train( + learner = learner, store_models = store_models, is_sequential = is_sequential + ) # predict for each set sets = sets[learner$predict_sets] @@ -283,6 +279,35 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, }, set = names(sets), row_ids = sets) pdatas = discard(pdatas, is.null) + # In case we are parallelizing and the learner needs marshaling, we need to send the marshaled model back to + # the main process. We can skip this if we are using callr encapsulation as we already have the marshaled + # model. + marshal_maybe_after_predict( + learner = learner, store_models = store_models, is_sequential = is_sequential, model_marshaled = model_marshaled_or_null + ) + + learner_state = set_class(learner$state, c("learner_state", "list")) + + list(learner_state = learner_state, prediction = pdatas, param_values = learner$param_set$values, learner_hash = learner_hash) +} + +unmarshal_maybe_after_train = function(learner, store_models, is_sequential) { + if (!is_marshaled_model(learner$model)) return(NULL) + if (is_sequential) { + # no need to keep the marshaled model in RAM, because we can send back the unmarshaled model back to the main + # process + model_marshaled = NULL + learner$model = unmarshal_model(model = learner$model, inplace = TRUE) + } else { + # here, we simultaneously keep the marshaled and unmarshaled model in RAM + model_marshaled = learner$model + learner$model = unmarshal_model(model = learner$model, inplace = FALSE) + } + + model_marshaled +} + +marshal_maybe_after_predict = function(learner, store_models, is_sequential, model_marshaled) { if (!store_models) { lg$debug("Erasing stored model for learner '%s'", learner$id) learner$state$model = NULL @@ -291,13 +316,8 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, learner$model = model_marshaled } else if (!is_sequential) { # parallelization without callr - learner$model = marshal_model(learner$model, clone = FALSE) + learner$model = marshal_model(learner$model, inplace = TRUE) } - # no parallelization and no callr --> nothing to do - # no parallelization and callr --> already unmarshaled - - - list(learner_state = learner$state, prediction = pdatas, param_values = learner$param_set$values, learner_hash = learner_hash) } append_log = function(log = NULL, stage = NA_character_, class = NA_character_, msg = character()) { diff --git a/man/marshaling.Rd b/man/marshaling.Rd index fc32d024c..a93d3fe95 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -61,8 +61,8 @@ Note that it is not guarateed that \code{is_marshaled_model(marshal_model(x))} r This is because the default \code{marshal_model(x)} returns \code{x} as-is. \item The \code{inplace} argument determines whether in-place marshaling should be performed. This is especially relevant in the context of references semantics. -If \code{inplace} is \code{FALSE}, the marshaled model should not share references with the original model. -If \code{inplace} is \code{TRUE} this is not necessary and the original object can be modified in-place. +If \code{inplace} is \code{FALSE}, the original input should not be modified, otherwise this is allowed. +Note that the input and output can still share references, even when \code{inplace} is \code{FALSE}. } } \section{Implementing Marshaling}{ diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index b5b53361d..a61f08592 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -189,6 +189,17 @@ test_that("parallel execution and callr marshal once", { expect_false(rr$learners[[1]]$marshaled) }) +test_that("marshaling works when store_models is FALSE", { + learner = lrn("classif.debug", count_marshaling = FALSE, encapsulate = c(train = "callr")) + task = tsk("iris") + resampling = rsmp("holdout") + rr = with_future(future::multisession, { + resample(task, learner, resampling, store_models = FALSE, unmarshal = TRUE) + }) + expect_resample_result(rr) + rr$learners[[1]]$model +}) + test_that("unmarshal parameter is respected", { learner = lrn("classif.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) From dfc345e20a0cb952ef6fbfcaae8100e7321c0f18 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 10 Apr 2024 18:00:22 +0200 Subject: [PATCH 34/47] inplace marshal for ResultData --- R/ResultData.R | 4 ++-- R/marshal.R | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/R/ResultData.R b/R/ResultData.R index 8d7828547..e6007fd1b 100644 --- a/R/ResultData.R +++ b/R/ResultData.R @@ -249,7 +249,7 @@ ResultData = R6Class("ResultData", #' Additional arguments passed to [`marshal_model()`]. marshal = function(...) { learner_state = NULL - self$data$fact[, learner_state := lapply(learner_state, function(x) marshal_state_if_model(.state = x, ...))] + self$data$fact[, learner_state := lapply(learner_state, function(x) marshal_state_if_model(.state = x, inplace = TRUE, ...))] invisible(self) }, #' @description @@ -259,7 +259,7 @@ ResultData = R6Class("ResultData", #' Additional arguments passed to [`unmarshal_model()`]. unmarshal = function(...) { learner_state = NULL - self$data$fact[, learner_state := lapply(learner_state, function(x) unmarshal_state_if_model(.state = x, ...))] + self$data$fact[, learner_state := lapply(learner_state, function(x) unmarshal_state_if_model(.state = x, inplace = TRUE, ...))] invisible(self) }, diff --git a/R/marshal.R b/R/marshal.R index cc2e9a945..7c92a9850 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -128,16 +128,16 @@ unmarshal_model.default = function(model, inplace = FALSE, ...) { model } -marshal_state_if_model = function(.state, ...) { +marshal_state_if_model = function(.state, inplace, ...) { if (!is.null(.state$model)) { - .state$model = marshal_model(.state$model, ...) + .state$model = marshal_model(.state$model, inplace, ...) } .state } -unmarshal_state_if_model = function(.state, ...) { +unmarshal_state_if_model = function(.state, inplace, ...) { if (!is.null(.state$model)) { - .state$model = unmarshal_model(model = .state$model, ...) + .state$model = unmarshal_model(model = .state$model, inplace = inplace, ...) } .state } From c979ba81863060b6da24f9ce33aa9c2865984be9 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 17 Apr 2024 09:21:08 +0200 Subject: [PATCH 35/47] fix class of marshaled classif debug --- R/LearnerClassifDebug.R | 2 +- tests/testthat/test_marshal.R | 2 +- tests/testthat/test_mlr_learners_classif_debug.R | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/R/LearnerClassifDebug.R b/R/LearnerClassifDebug.R index 3f41d5067..7918117b4 100644 --- a/R/LearnerClassifDebug.R +++ b/R/LearnerClassifDebug.R @@ -228,7 +228,7 @@ marshal_model.classif.debug_model = function(model, ...) { if (!is.null(model$marshal_count)) { model$marshal_count = model$marshal_count + 1 } - newclass = c("regr.debug_model_marshaled", "marshaled") + newclass = c("classif.debug_model_marshaled", "marshaled") structure(list(marshaled = model, packages = "mlr3"), class = newclass) } diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R index 086d0fe6f..4d90be6df 100644 --- a/tests/testthat/test_marshal.R +++ b/tests/testthat/test_marshal.R @@ -68,5 +68,5 @@ test_that("marshal count works for LearnerClassifDebug", { }) test_that("printer", { - expect_output(print(lrn("classif.debug")$train(tsk("iris"))$marshal()$model), "") + expect_output(print(lrn("classif.debug")$train(tsk("iris"))$marshal()$model), "") }) diff --git a/tests/testthat/test_mlr_learners_classif_debug.R b/tests/testthat/test_mlr_learners_classif_debug.R index 8dae61430..48ffb4bde 100644 --- a/tests/testthat/test_mlr_learners_classif_debug.R +++ b/tests/testthat/test_mlr_learners_classif_debug.R @@ -71,3 +71,13 @@ test_that("default_values works with empty search space", { learner = lrn("classif.debug") expect_list(default_values(learner, ps(), task), len = 0) }) + +test_that("marshaling", { + l = lrn("classif.debug") + expect_learner(l, tsk("iris")) + task = tsk("iris") + l$train(task) + p1 = l$predict(task) + p2 = l$marshal()$unmarshal()$predict(task) + expect_equal(p1, p2) +}) From 7f637c8a610ff03621acd357c6c771a8c0190c78 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 17 Apr 2024 10:29:49 +0200 Subject: [PATCH 36/47] add class to learner state for marshaling --- NAMESPACE | 2 ++ R/Learner.R | 20 ++++++++++++++++++++ R/LearnerClassifDebug.R | 6 ++++-- R/worker.R | 4 ++-- tests/testthat/test_Learner.R | 7 +++++++ 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 7464c9018..62c2aa96f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -89,6 +89,7 @@ S3method(is_missing_prediction_data,PredictionDataClassif) S3method(is_missing_prediction_data,PredictionDataRegr) S3method(marshal_model,classif.debug_model) S3method(marshal_model,default) +S3method(marshal_model,learner_state) S3method(partition,Task) S3method(partition,TaskClassif) S3method(partition,TaskRegr) @@ -109,6 +110,7 @@ S3method(summary,Task) S3method(tail,Task) S3method(unmarshal_model,classif.debug_model_marshaled) S3method(unmarshal_model,default) +S3method(unmarshal_model,learner_state_marshaled) S3method(unmarshal_model,marshaled) export(BenchmarkResult) export(DataBackend) diff --git a/R/Learner.R b/R/Learner.R index 91e404f55..3e8a93d80 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -580,3 +580,23 @@ default_values.Learner = function(x, search_space, task, ...) { # nolint # sprintf("", x$id) # } + +#' @export +marshal_model.learner_state = function(model, inplace = FALSE, ...) { + mm = marshal_model(model$model, inplace = inplace, ...) + if (!is_marshaled_model(mm)) { + return(model) + } + model$model = mm + structure(list( + marshaled = model, + packages = "mlr3" + ), class = c("learner_state_marshaled", "list_marshaled", "marshaled")) +} + +#' @export +unmarshal_model.learner_state_marshaled = function(model, inplace = FALSE, ...) { + mm = model$marshaled + mm$model = unmarshal_model(mm$model, inplace = inplace, ...) + return(mm) +} diff --git a/R/LearnerClassifDebug.R b/R/LearnerClassifDebug.R index 7918117b4..a498a3e58 100644 --- a/R/LearnerClassifDebug.R +++ b/R/LearnerClassifDebug.R @@ -228,8 +228,10 @@ marshal_model.classif.debug_model = function(model, ...) { if (!is.null(model$marshal_count)) { model$marshal_count = model$marshal_count + 1 } - newclass = c("classif.debug_model_marshaled", "marshaled") - structure(list(marshaled = model, packages = "mlr3"), class = newclass) + structure(list( + marshaled = model, packages = "mlr3"), + class = c("classif.debug_model_marshaled", "marshaled") + ) } #' @export diff --git a/R/worker.R b/R/worker.R index 9121013c2..2e5581ba8 100644 --- a/R/worker.R +++ b/R/worker.R @@ -74,7 +74,7 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL # unmarshal_model does nothing if the model was not marshaled - learner$state = insert_named(learner$state, list( + learner$state = set_class(insert_named(learner$state, list( model = result$result, log = log, train_time = train_time, @@ -82,7 +82,7 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL task_hash = task$hash, feature_names = task$feature_names, mlr3_version = mlr_reflections$package_version - )) + )), c("learner_state", "list")) if (is.null(result$result)) { lg$debug("Learner '%s' on task '%s' failed to %s a model", diff --git a/tests/testthat/test_Learner.R b/tests/testthat/test_Learner.R index 910fafde0..c6557edc6 100644 --- a/tests/testthat/test_Learner.R +++ b/tests/testthat/test_Learner.R @@ -341,3 +341,10 @@ test_that("marshaling and encapsulation", { learner$train(task) expect_equal(learner$model$marshal_count, 0) }) + +test_that("marshal state", { + state = lrn("classif.debug")$train(tsk("iris"))$state + sm = marshal_model(state) + expect_true(is_marshaled_model(sm)) + expect_equal(state, unmarshal_model(marshal_model(state))) +}) From 688f34c6380ded34700b51d320f8db0223e50cfc Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 17 Apr 2024 12:42:42 +0200 Subject: [PATCH 37/47] ... --- R/Learner.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/R/Learner.R b/R/Learner.R index 3e8a93d80..45c2b442a 100644 --- a/R/Learner.R +++ b/R/Learner.R @@ -583,6 +583,9 @@ default_values.Learner = function(x, search_space, task, ...) { # nolint #' @export marshal_model.learner_state = function(model, inplace = FALSE, ...) { + if (is.null(model$model)) { + return(model) + } mm = marshal_model(model$model, inplace = inplace, ...) if (!is_marshaled_model(mm)) { return(model) From 626a3df18c76588a4285e7eba9395119c9528245 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 17 Apr 2024 14:00:37 +0200 Subject: [PATCH 38/47] ... --- R/LearnerClassifDebug.R | 8 ++++---- R/LearnerRegrDebug.R | 14 +++++++------- R/marshal.R | 17 ++++++++--------- man/marshaling.Rd | 17 ++++++++--------- man/mlr_learners_classif.debug.Rd | 4 ++-- man/mlr_learners_regr.debug.Rd | 17 +++-------------- 6 files changed, 32 insertions(+), 45 deletions(-) diff --git a/R/LearnerClassifDebug.R b/R/LearnerClassifDebug.R index 9baf167b8..3ca1953e7 100644 --- a/R/LearnerClassifDebug.R +++ b/R/LearnerClassifDebug.R @@ -81,14 +81,14 @@ LearnerClassifDebug = R6Class("LearnerClassifDebug", inherit = LearnerClassif, ) }, #' @description - #' Marshals the learner. + #' Marshal the learner's model. #' @param ... (any)\cr #' Additional arguments passed to [`marshal_model()`]. marshal = function(...) { learner_marshal(.learner = self, ...) }, #' @description - #' Unmarshal the learner. + #' Unmarshal the learner's model. #' @param ... (any)\cr #' Additional arguments passed to [`unmarshal_model()`]. unmarshal = function(...) { @@ -224,7 +224,7 @@ mlr_learners$add("classif.debug", function() LearnerClassifDebug$new()) #' @export #' @method marshal_model classif.debug_model -marshal_model.classif.debug_model = function(model, ...) { +marshal_model.classif.debug_model = function(model, inplace = FALSE, ...) { if (!is.null(model$marshal_count)) { model$marshal_count = model$marshal_count + 1 } @@ -236,6 +236,6 @@ marshal_model.classif.debug_model = function(model, ...) { #' @export #' @method unmarshal_model classif.debug_model_marshaled -unmarshal_model.classif.debug_model_marshaled = function(model, ...) { +unmarshal_model.classif.debug_model_marshaled = function(model, inplace = FALSE, ...) { model$marshaled } diff --git a/R/LearnerRegrDebug.R b/R/LearnerRegrDebug.R index f8be7dc49..b368b317c 100644 --- a/R/LearnerRegrDebug.R +++ b/R/LearnerRegrDebug.R @@ -33,17 +33,17 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, #' @description #' Creates a new instance of this [R6][R6::R6Class] class. initialize = function() { - param_set = ps( - predict_missing = p_dbl(0, 1, default = 0, tags = "predict"), - predict_missing_type = p_fct(c("na", "omit"), default = "na", tags = "predict"), - save_tasks = p_lgl(default = FALSE, tags = c("train", "predict")), - threads = p_int(1L, tags = c("train", "threads")), - x = p_dbl(0, 1, tags = "train") - ) super$initialize( id = "regr.debug", feature_types = c("logical", "integer", "numeric", "character", "factor", "ordered"), predict_types = c("response", "se"), + param_set = ps( + predict_missing = p_dbl(0, 1, default = 0, tags = "predict"), + predict_missing_type = p_fct(c("na", "omit"), default = "na", tags = "predict"), + save_tasks = p_lgl(default = FALSE, tags = c("train", "predict")), + threads = p_int(1L, tags = c("train", "threads")), + x = p_dbl(0, 1, tags = "train") + ), param_set = param_set, properties = "missings", man = "mlr3::mlr_learners_regr.debug", diff --git a/R/marshal.R b/R/marshal.R index 7c92a9850..c9b46f52e 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -13,15 +13,16 @@ #' * the S3 generic `marshal_model(model, inplace, ...)`. #' Which takes in a model and returns it in marshaled form. #' The marshaled object should be a list with named elements `marshaled` and `packages`, where the former contains -#' the marshaled object, and the latter the packages required to unmarshal it. -#' This list should have the classes of the original object with the suffix `"_marshaled"` appended and the +#' the marshaled object, and the latter the package that contains the packages requrired to unmarshal. +#' Most importantly, this list should contain the package that contains the `unmarshal_model` method. +#' This `$marshaled` field should have the classes of the original object with the suffix `"_marshaled"` appended and the #' root class should be set to `"marshaled"`. #' * the S3 generic `unmarshal_model(model, inplace ...)`. #' Which takes in the marshaled model and returns it in unmarshaled form. #' The generic takes care that the packages specified during `"marshal"` are loaded, and errs if they are not. -#' The returned object must not inherit from class `"marshaled"`. -#' * the function `is_ marshaled_model(model)`, which returns `TRUE` if the model inherits from class `"marshaled"` -#' and `FALSE` otherwise. +#' The returned object must not inherit from class `"marshaled"` and should reconstrct the original object. +#' * the function `is_marshaled_model(model)`. +#' This (helper) function returns `TRUE` if the model inherits from class `"marshaled"` and `FALSE` otherwise. #' #' The contract of these generics is: #' * `unmarshal_model(marshal_model(x))` returns `x` as is. @@ -36,15 +37,13 @@ #' #' @section Implementing Marshaling: #' -#' In order to implement marshaling for a Learner, you only need to overload the `marshal_model` and `unmarshal_model` -#' methods and tag the learner with the `"marshal"` property accordingly. +#' In order to implement marshaling for a Learner, you only to overload the `marshal_model` and `unmarshal_model` +#' methods for the class of the learner's model and tag the learner with the `"marshal"` property. #' #' To make marshaling accessible in an R6-manner, you should also add the public methods `$marshal()`, `$unmarshal()` #' and the active binding `$marshaled`. #' To make this as convenient as possible, the functions `learner_marshal(learner)`, `learner_unmarshal(learner)` #' and `learner_marshaled(learner)` are provided and can be called from within the public methods. -#' All three functions throw an error if the learner is not trained and otherwise call -#' `marshal_model()`, `unmarshal_model()` or `is_marshaled_model()` on the learner's model. #' #' You can verify whether you have correctly implemented marshaling by using the internal test helper #' `expect_marshalable_learner()`. This is also run by `expect_learner()` if a task is provided. diff --git a/man/marshaling.Rd b/man/marshaling.Rd index a93d3fe95..1f5c9a53b 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -41,15 +41,16 @@ The central functions (and the only methods that are used by \code{mlr3} interna \item the S3 generic \code{marshal_model(model, inplace, ...)}. Which takes in a model and returns it in marshaled form. The marshaled object should be a list with named elements \code{marshaled} and \code{packages}, where the former contains -the marshaled object, and the latter the packages required to unmarshal it. -This list should have the classes of the original object with the suffix \code{"_marshaled"} appended and the +the marshaled object, and the latter the package that contains the packages requrired to unmarshal. +Most importantly, this list should contain the package that contains the \code{unmarshal_model} method. +This \verb{$marshaled} field should have the classes of the original object with the suffix \code{"_marshaled"} appended and the root class should be set to \code{"marshaled"}. \item the S3 generic \verb{unmarshal_model(model, inplace ...)}. Which takes in the marshaled model and returns it in unmarshaled form. The generic takes care that the packages specified during \code{"marshal"} are loaded, and errs if they are not. -The returned object must not inherit from class \code{"marshaled"}. -\item the function \verb{is_ marshaled_model(model)}, which returns \code{TRUE} if the model inherits from class \code{"marshaled"} -and \code{FALSE} otherwise. +The returned object must not inherit from class \code{"marshaled"} and should reconstrct the original object. +\item the function \code{is_marshaled_model(model)}. +This (helper) function returns \code{TRUE} if the model inherits from class \code{"marshaled"} and \code{FALSE} otherwise. } The contract of these generics is: @@ -68,15 +69,13 @@ Note that the input and output can still share references, even when \code{inpla \section{Implementing Marshaling}{ -In order to implement marshaling for a Learner, you only need to overload the \code{marshal_model} and \code{unmarshal_model} -methods and tag the learner with the \code{"marshal"} property accordingly. +In order to implement marshaling for a Learner, you only to overload the \code{marshal_model} and \code{unmarshal_model} +methods for the class of the learner's model and tag the learner with the \code{"marshal"} property. To make marshaling accessible in an R6-manner, you should also add the public methods \verb{$marshal()}, \verb{$unmarshal()} and the active binding \verb{$marshaled}. To make this as convenient as possible, the functions \code{learner_marshal(learner)}, \code{learner_unmarshal(learner)} and \code{learner_marshaled(learner)} are provided and can be called from within the public methods. -All three functions throw an error if the learner is not trained and otherwise call -\code{marshal_model()}, \code{unmarshal_model()} or \code{is_marshaled_model()} on the learner's model. You can verify whether you have correctly implemented marshaling by using the internal test helper \code{expect_marshalable_learner()}. This is also run by \code{expect_learner()} if a task is provided. diff --git a/man/mlr_learners_classif.debug.Rd b/man/mlr_learners_classif.debug.Rd index 9c1d9b404..f88bdd5ef 100644 --- a/man/mlr_learners_classif.debug.Rd +++ b/man/mlr_learners_classif.debug.Rd @@ -163,7 +163,7 @@ Creates a new instance of this \link[R6:R6Class]{R6} class. \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-LearnerClassifDebug-marshal}{}}} \subsection{Method \code{marshal()}}{ -Marshals the learner. +Marshal the learner's model. \subsection{Usage}{ \if{html}{\out{
}}\preformatted{LearnerClassifDebug$marshal(...)}\if{html}{\out{
}} } @@ -181,7 +181,7 @@ Additional arguments passed to \code{\link[=marshal_model]{marshal_model()}}.} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-LearnerClassifDebug-unmarshal}{}}} \subsection{Method \code{unmarshal()}}{ -Unmarshal the learner. +Unmarshal the learner's model. \subsection{Usage}{ \if{html}{\out{
}}\preformatted{LearnerClassifDebug$unmarshal(...)}\if{html}{\out{
}} } diff --git a/man/mlr_learners_regr.debug.Rd b/man/mlr_learners_regr.debug.Rd index 2c2c53b23..18d82fe18 100644 --- a/man/mlr_learners_regr.debug.Rd +++ b/man/mlr_learners_regr.debug.Rd @@ -27,23 +27,12 @@ lrn("regr.debug") \section{Meta Information}{ -\itemize{ -\item Task type: \dQuote{regr} -\item Predict Types: \dQuote{response}, \dQuote{se} -\item Feature Types: \dQuote{logical}, \dQuote{integer}, \dQuote{numeric}, \dQuote{character}, \dQuote{factor}, \dQuote{ordered} -\item Required Packages: \CRANpkg{mlr3} -} +\verb{r mlr3misc::rd_info(mlr3::lrn("regr.debug"))} } \section{Parameters}{ -\tabular{lllll}{ - Id \tab Type \tab Default \tab Levels \tab Range \cr - predict_missing \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr - predict_missing_type \tab character \tab na \tab na, omit \tab - \cr - save_tasks \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr - threads \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr - x \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr -} + +\verb{r mlr3misc::rd_info(mlr3::lrn("regr.debug")$param_set)} } \examples{ From 5ada07cb2dc4d00f0a4b721fe90bfcbc0bfd4113 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Wed, 17 Apr 2024 15:05:03 +0200 Subject: [PATCH 39/47] typo --- R/LearnerRegrDebug.R | 1 - man/mlr_learners_regr.debug.Rd | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/R/LearnerRegrDebug.R b/R/LearnerRegrDebug.R index b368b317c..a2396baad 100644 --- a/R/LearnerRegrDebug.R +++ b/R/LearnerRegrDebug.R @@ -44,7 +44,6 @@ LearnerRegrDebug = R6Class("LearnerRegrDebug", inherit = LearnerRegr, threads = p_int(1L, tags = c("train", "threads")), x = p_dbl(0, 1, tags = "train") ), - param_set = param_set, properties = "missings", man = "mlr3::mlr_learners_regr.debug", data_formats = c("data.table", "Matrix"), diff --git a/man/mlr_learners_regr.debug.Rd b/man/mlr_learners_regr.debug.Rd index 18d82fe18..2c2c53b23 100644 --- a/man/mlr_learners_regr.debug.Rd +++ b/man/mlr_learners_regr.debug.Rd @@ -27,12 +27,23 @@ lrn("regr.debug") \section{Meta Information}{ -\verb{r mlr3misc::rd_info(mlr3::lrn("regr.debug"))} +\itemize{ +\item Task type: \dQuote{regr} +\item Predict Types: \dQuote{response}, \dQuote{se} +\item Feature Types: \dQuote{logical}, \dQuote{integer}, \dQuote{numeric}, \dQuote{character}, \dQuote{factor}, \dQuote{ordered} +\item Required Packages: \CRANpkg{mlr3} +} } \section{Parameters}{ - -\verb{r mlr3misc::rd_info(mlr3::lrn("regr.debug")$param_set)} +\tabular{lllll}{ + Id \tab Type \tab Default \tab Levels \tab Range \cr + predict_missing \tab numeric \tab 0 \tab \tab \eqn{[0, 1]}{[0, 1]} \cr + predict_missing_type \tab character \tab na \tab na, omit \tab - \cr + save_tasks \tab logical \tab FALSE \tab TRUE, FALSE \tab - \cr + threads \tab integer \tab - \tab \tab \eqn{[1, \infty)}{[1, Inf)} \cr + x \tab numeric \tab - \tab \tab \eqn{[0, 1]}{[0, 1]} \cr +} } \examples{ From 3d64b1a86e3c52d79f682b1e8dbb6e96c59d6617 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Mon, 22 Apr 2024 13:40:04 +0200 Subject: [PATCH 40/47] cleanup marshaling --- NAMESPACE | 1 - NEWS.md | 2 +- R/LearnerClassifDebug.R | 2 +- R/benchmark.R | 3 +- R/marshal.R | 34 ++++++-------- R/resample.R | 2 +- R/worker.R | 69 +++++++++++++++-------------- inst/testthat/helper_expectations.R | 5 --- man-roxygen/param_unmarshal.R | 4 +- man/benchmark.Rd | 4 +- man/marshaling.Rd | 27 +++++------ man/mlr_learners_classif.debug.Rd | 2 +- man/resample.Rd | 4 +- tests/testthat/test_marshal.R | 4 +- tests/testthat/test_resample.R | 23 +++++++++- 15 files changed, 98 insertions(+), 88 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 62c2aa96f..6cac251b1 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -111,7 +111,6 @@ S3method(tail,Task) S3method(unmarshal_model,classif.debug_model_marshaled) S3method(unmarshal_model,default) S3method(unmarshal_model,learner_state_marshaled) -S3method(unmarshal_model,marshaled) export(BenchmarkResult) export(DataBackend) export(DataBackendDataTable) diff --git a/NEWS.md b/NEWS.md index b5065898e..c45b7fe29 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ * Feat: added support for `"marshal"` property, which allows learners to process models so they can be serialized. This happens automatically during `resample()` -and `benchmark()`. The naming was inspired by the {marshal} package. +and `benchmark()`. * Log encapsulated errors and warnings with the `lgr` package. # mlr3 0.18.0 diff --git a/R/LearnerClassifDebug.R b/R/LearnerClassifDebug.R index 3ca1953e7..3fe97a9ed 100644 --- a/R/LearnerClassifDebug.R +++ b/R/LearnerClassifDebug.R @@ -24,7 +24,7 @@ #' \item{warning_train:}{Probability to signal a warning during train.} #' \item{x:}{Numeric tuning parameter. Has no effect.} #' \item{iter:}{Integer parameter for testing hotstarting.} -#' \item{count_marshaling:}{If `TRUE`, `marshal_model` will increase the `marshal_count` by 1. This value is initialized to `FALSE`.} +#' \item{count_marshaling:}{If `TRUE`, `marshal_model` will increase the `marshal_count` by 1 each time it is called. The default is `FALSE`.} #' } #' Note that segfaults may not be triggered reliably on your operating system. #' Also note that if they work as intended, they will tear down your R session immediately! diff --git a/R/benchmark.R b/R/benchmark.R index 462aeb64e..41555f1b1 100644 --- a/R/benchmark.R +++ b/R/benchmark.R @@ -82,6 +82,7 @@ benchmark = function(design, store_models = FALSE, store_backends = TRUE, encaps assert_subset(clone, c("task", "learner", "resampling")) assert_data_frame(design, min.rows = 1L) assert_names(names(design), must.include = c("task", "learner", "resampling")) + assert_flag(unmarshal) design$task = list(assert_tasks(as_tasks(design$task))) design$learner = list(assert_learners(as_learners(design$learner))) design$resampling = list(assert_resamplings(as_resamplings(design$resampling), instantiated = TRUE)) @@ -184,7 +185,7 @@ benchmark = function(design, store_models = FALSE, store_backends = TRUE, encaps res = future_map(n, workhorse, task = grid$task, learner = grid$learner, resampling = grid$resampling, iteration = grid$iteration, param_values = grid$param_values, mode = grid$mode, - MoreArgs = list(store_models = store_models, lgr_threshold = lgr_threshold, pb = pb) + MoreArgs = list(store_models = store_models, lgr_threshold = lgr_threshold, pb = pb, unmarshal = unmarshal) ) grid = insert_named(grid, list( diff --git a/R/marshal.R b/R/marshal.R index c9b46f52e..cce744931 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -6,47 +6,44 @@ #' Marshaling is the process of processing the model of a trained [`Learner`] so it an be successfully serialized and #' deserialized. The naming is inspired by the [marshal package](https://github.com/HenrikBengtsson/marshal) and we #' plan to fully migrate to this package once it is on CRAN. -#' The supported implementation until then should therfore be considered as a temporary solution and is likely +#' The current implementation should therfore be considered as a temporary solution and is likely #' to change in the future. #' #' The central functions (and the only methods that are used by `mlr3` internally) are: #' * the S3 generic `marshal_model(model, inplace, ...)`. #' Which takes in a model and returns it in marshaled form. +#' This means, that the resulting object can be serialized and de-serialzed without loss of information. #' The marshaled object should be a list with named elements `marshaled` and `packages`, where the former contains #' the marshaled object, and the latter the package that contains the packages requrired to unmarshal. #' Most importantly, this list should contain the package that contains the `unmarshal_model` method. -#' This `$marshaled` field should have the classes of the original object with the suffix `"_marshaled"` appended and the +#' The returned object should have the classes of the original object with the suffix `"_marshaled"` appended and the #' root class should be set to `"marshaled"`. +#' Note that it is not guarateed that `is_marshaled_model(marshal_model(x))` returns `TRUE`. +#' This is because the default `marshal_model(x)` returns `x` as-is. #' * the S3 generic `unmarshal_model(model, inplace ...)`. #' Which takes in the marshaled model and returns it in unmarshaled form. #' The generic takes care that the packages specified during `"marshal"` are loaded, and errs if they are not. -#' The returned object must not inherit from class `"marshaled"` and should reconstrct the original object. +#' Calling this function on a marshaled model should reconstruct the original model, i.e. +#' `unmarshal_model(marshal_model(x))` should return `x` as is. #' * the function `is_marshaled_model(model)`. #' This (helper) function returns `TRUE` if the model inherits from class `"marshaled"` and `FALSE` otherwise. #' -#' The contract of these generics is: -#' * `unmarshal_model(marshal_model(x))` returns `x` as is. -#' * `unmarshal_model(x)` can be serialized and de-serialzed without loss of information. -#' * If `is_marshaled_model(x)` is `TRUE`, this means that `x` is in marshaled form. -#' Note that it is not guarateed that `is_marshaled_model(marshal_model(x))` returns `TRUE`. -#' This is because the default `marshal_model(x)` returns `x` as-is. -#' * The `inplace` argument determines whether in-place marshaling should be performed. -#' This is especially relevant in the context of references semantics. -#' If `inplace` is `FALSE`, the original input should not be modified, otherwise this is allowed. -#' Note that the input and output can still share references, even when `inplace` is `FALSE`. +#' For both `marshal_model` and `unmarshal_model`, the `inplace` argument determines whether in-place marshaling +#' should be performed. This is especially relevant in the context of references semantics. +#' If `inplace` is `FALSE`, the original input should not be modified, otherwise this is allowed. +#' Note that the input and output can still share references, even when `inplace` is `FALSE`. #' #' @section Implementing Marshaling: #' -#' In order to implement marshaling for a Learner, you only to overload the `marshal_model` and `unmarshal_model` +#' In order to implement marshaling for a Learner, you need to overload the `marshal_model` and `unmarshal_model` #' methods for the class of the learner's model and tag the learner with the `"marshal"` property. -#' #' To make marshaling accessible in an R6-manner, you should also add the public methods `$marshal()`, `$unmarshal()` #' and the active binding `$marshaled`. #' To make this as convenient as possible, the functions `learner_marshal(learner)`, `learner_unmarshal(learner)` #' and `learner_marshaled(learner)` are provided and can be called from within the public methods. #' #' You can verify whether you have correctly implemented marshaling by using the internal test helper -#' `expect_marshalable_learner()`. This is also run by `expect_learner()` if a task is provided. +#' `expect_marshalable_learner(learner, task)`. This is also run by `expect_learner()` if a task is provided. #' #' For a concrete example on how to implement marshaling, see [`LearnerClassifDebug`]. #' @@ -117,11 +114,6 @@ marshal_model.default = function(model, ...) { model } -#' @export -unmarshal_model.marshaled = function(model, inplace = FALSE,...) { - model[["marshaled"]] -} - #' @export unmarshal_model.default = function(model, inplace = FALSE, ...) { model diff --git a/R/resample.R b/R/resample.R index 4a68a0831..6c22bd24a 100644 --- a/R/resample.R +++ b/R/resample.R @@ -113,7 +113,7 @@ resample = function(task, learner, resampling, store_models = FALSE, store_backe } res = future_map(n, workhorse, iteration = seq_len(n), learner = grid$learner, mode = grid$mode, - MoreArgs = list(task = task, resampling = resampling, store_models = store_models, lgr_threshold = lgr_threshold, pb = pb) + MoreArgs = list(task = task, resampling = resampling, store_models = store_models, lgr_threshold = lgr_threshold, pb = pb, unmarshal = unmarshal) ) data = data.table( diff --git a/R/worker.R b/R/worker.R index 379d8d277..a8f8bdbfb 100644 --- a/R/worker.R +++ b/R/worker.R @@ -1,4 +1,4 @@ -learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NULL, mode = "train", unmarshal = TRUE) { +learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NULL, mode = "train") { # This wrapper calls learner$.train, and additionally performs some basic # checks that the training was successful. # Exceptions here are possibly encapsulated, so that they get captured @@ -72,8 +72,6 @@ learner_train = function(learner, task, train_row_ids = NULL, test_row_ids = NUL log = append_log(NULL, "train", result$log$class, result$log$msg) train_time = result$elapsed - # unmarshal_model does nothing if the model was not marshaled - learner$state = set_class(insert_named(learner$state, list( model = result$result, log = log, @@ -223,7 +221,7 @@ learner_predict = function(learner, task, row_ids = NULL) { } -workhorse = function(iteration, task, learner, resampling, param_values = NULL, lgr_threshold, store_models = FALSE, pb = NULL, mode = "train", is_sequential = TRUE) { +workhorse = function(iteration, task, learner, resampling, param_values = NULL, lgr_threshold, store_models = FALSE, pb = NULL, mode = "train", is_sequential = TRUE, unmarshal = TRUE) { if (!is.null(pb)) { pb(sprintf("%s|%s|i:%i", task$id, learner$id, iteration)) } @@ -259,17 +257,17 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, learner$param_set$set_values(.values = param_values) } learner_hash = learner$hash - learner = learner_train(learner, task, sets[["train"]], sets[["test"]], mode = mode, unmarshal = FALSE) + learner = learner_train(learner, task, sets[["train"]], sets[["test"]], mode = mode) # The model is in marshaled form in case we did callr encapsulation and the model needed marshaling. - # But for the predict step we need it unmarshaled. - # In case we have a marshaled model and are parallelizing, we avoid to marshal again after unmarshaling - # the model for the predict step. - # We do this by temporarily keep the marshaled and unmarshaled model in RAM (the marshaled model model of course only - # if store_models is TRUE) - model_marshaled_or_null = unmarshal_maybe_after_train( - learner = learner, store_models = store_models, is_sequential = is_sequential + # For the predict step we need it unmarshaled. + # If the model is later sent back to main process (when parallelizing) and needs marshaling and the model is already + # in marshaled form, we preserve the marshaled model. + # This means, we temporarily keep both the marshaled and unmarshaled model in RAM + model_marshaled_or_null = get_marshaled_if_needed( + learner = learner, store_models = store_models, is_sequential = is_sequential, unmarshal = unmarshal ) + learner$model = unmarshal_model(model = learner$model, inplace = FALSE) # predict for each set sets = sets[learner$predict_sets] @@ -279,11 +277,11 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, }, set = names(sets), row_ids = sets) pdatas = discard(pdatas, is.null) - # In case we are parallelizing and the learner needs marshaling, we need to send the marshaled model back to - # the main process. We can skip this if we are using callr encapsulation as we already have the marshaled - # model. - marshal_maybe_after_predict( - learner = learner, store_models = store_models, is_sequential = is_sequential, model_marshaled = model_marshaled_or_null + # In case store_models is FALSE, we now discard the model + # Otherwise, we set the model to the marshaled model, if callr encapsulation was used and we are parallelizing + # If callr encapsulation was not used, we still have to marshal the model + set_model_after_predict( + learner = learner, store_models = store_models, is_sequential = is_sequential, model = model_marshaled_or_null ) learner_state = set_class(learner$state, c("learner_state", "list")) @@ -291,29 +289,34 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, list(learner_state = learner_state, prediction = pdatas, param_values = learner$param_set$values, learner_hash = learner_hash) } -unmarshal_maybe_after_train = function(learner, store_models, is_sequential) { - if (!is_marshaled_model(learner$model)) return(NULL) - if (is_sequential) { - # no need to keep the marshaled model in RAM, because we can send back the unmarshaled model back to the main - # process - model_marshaled = NULL - learner$model = unmarshal_model(model = learner$model, inplace = TRUE) - } else { - # here, we simultaneously keep the marshaled and unmarshaled model in RAM - model_marshaled = learner$model - learner$model = unmarshal_model(model = learner$model, inplace = FALSE) +get_marshaled_if_needed = function(learner, store_models, is_sequential, unmarshal) { + if (!is_marshaled_model(learner$model)) { + NULL + } else if (!store_models) { # else condition means model was marshaled + # not storing models, so no need to keep the marhshaled model + NULL + } else if (is_sequential && unmarshal) { + # because of no parallelization, we can send the unmarshaled model back, which the user + # wants anyway (because unmarshal = TRUE) + NULL + } else if (is_sequential && !unmarshal) { + # here, the user set unmarshal to FALSE, i.e. he wants the marshaled model, so we + # keep it, as we have already computed it + learner$model + } else { # parallel execution + # here, we just need to send back the marshaled model + learner$model } - - model_marshaled } -marshal_maybe_after_predict = function(learner, store_models, is_sequential, model_marshaled) { +set_model_after_predict = function(learner, store_models, is_sequential, model) { if (!store_models) { lg$debug("Erasing stored model for learner '%s'", learner$id) learner$state$model = NULL - } else if (!is.null(model_marshaled)) { + } else if (!is.null(model)) { # callr + parallelization - learner$model = model_marshaled + # or callr + sequential, but unmarshal was FALSE + learner$model = model } else if (!is_sequential) { # parallelization without callr learner$model = marshal_model(learner$model, inplace = TRUE) diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index 5465cc152..eb7079198 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -439,11 +439,6 @@ expect_marshalable_learner = function(learner, task) { expect_false(learner$marshaled) expect_equal(class(learner$model), class_prev) - - # when re-training, marshaled is reset - learner$predict(task) - expect_false(learner$train(task)$marshaled) - } expect_resampling = function(r, task = NULL) { diff --git a/man-roxygen/param_unmarshal.R b/man-roxygen/param_unmarshal.R index c950d6184..1776bf97d 100644 --- a/man-roxygen/param_unmarshal.R +++ b/man-roxygen/param_unmarshal.R @@ -1,2 +1,4 @@ #' @param unmarshal [`Learner`]\cr -#' Whether to unmarshal the learner. +#' Whether to unmarshal learners that were sent back to the main process in marshaled form. +#' Setting this to `TRUE` does not guarantee that all learners that require marshaling are marshaled. +#' To achieve this, call `$marshal()` on the result object. diff --git a/man/benchmark.Rd b/man/benchmark.Rd index 296aceb68..064f01094 100644 --- a/man/benchmark.Rd +++ b/man/benchmark.Rd @@ -60,7 +60,9 @@ providing a set with possible values \code{"task"}, \code{"learner"} and Per default, all input objects are cloned.} \item{unmarshal}{\code{\link{Learner}}\cr -Whether to unmarshal the learner.} +Whether to unmarshal learners that were sent back to the main process in marshaled form. +Setting this to \code{TRUE} does not guarantee that all learners that require marshaling are marshaled. +To achieve this, call \verb{$marshal()} on the result object.} } \value{ \link{BenchmarkResult}. diff --git a/man/marshaling.Rd b/man/marshaling.Rd index 1f5c9a53b..5843e7525 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -33,52 +33,47 @@ Additional parameters, currently unused.} Marshaling is the process of processing the model of a trained \code{\link{Learner}} so it an be successfully serialized and deserialized. The naming is inspired by the \href{https://github.com/HenrikBengtsson/marshal}{marshal package} and we plan to fully migrate to this package once it is on CRAN. -The supported implementation until then should therfore be considered as a temporary solution and is likely +The current implementation should therfore be considered as a temporary solution and is likely to change in the future. The central functions (and the only methods that are used by \code{mlr3} internally) are: \itemize{ \item the S3 generic \code{marshal_model(model, inplace, ...)}. Which takes in a model and returns it in marshaled form. +This means, that the resulting object can be serialized and de-serialzed without loss of information. The marshaled object should be a list with named elements \code{marshaled} and \code{packages}, where the former contains the marshaled object, and the latter the package that contains the packages requrired to unmarshal. Most importantly, this list should contain the package that contains the \code{unmarshal_model} method. -This \verb{$marshaled} field should have the classes of the original object with the suffix \code{"_marshaled"} appended and the +The returned object should have the classes of the original object with the suffix \code{"_marshaled"} appended and the root class should be set to \code{"marshaled"}. +Note that it is not guarateed that \code{is_marshaled_model(marshal_model(x))} returns \code{TRUE}. +This is because the default \code{marshal_model(x)} returns \code{x} as-is. \item the S3 generic \verb{unmarshal_model(model, inplace ...)}. Which takes in the marshaled model and returns it in unmarshaled form. The generic takes care that the packages specified during \code{"marshal"} are loaded, and errs if they are not. -The returned object must not inherit from class \code{"marshaled"} and should reconstrct the original object. +Calling this function on a marshaled model should reconstruct the original model, i.e. +\code{unmarshal_model(marshal_model(x))} should return \code{x} as is. \item the function \code{is_marshaled_model(model)}. This (helper) function returns \code{TRUE} if the model inherits from class \code{"marshaled"} and \code{FALSE} otherwise. } -The contract of these generics is: -\itemize{ -\item \code{unmarshal_model(marshal_model(x))} returns \code{x} as is. -\item \code{unmarshal_model(x)} can be serialized and de-serialzed without loss of information. -\item If \code{is_marshaled_model(x)} is \code{TRUE}, this means that \code{x} is in marshaled form. -Note that it is not guarateed that \code{is_marshaled_model(marshal_model(x))} returns \code{TRUE}. -This is because the default \code{marshal_model(x)} returns \code{x} as-is. -\item The \code{inplace} argument determines whether in-place marshaling should be performed. -This is especially relevant in the context of references semantics. +For both \code{marshal_model} and \code{unmarshal_model}, the \code{inplace} argument determines whether in-place marshaling +should be performed. This is especially relevant in the context of references semantics. If \code{inplace} is \code{FALSE}, the original input should not be modified, otherwise this is allowed. Note that the input and output can still share references, even when \code{inplace} is \code{FALSE}. } -} \section{Implementing Marshaling}{ -In order to implement marshaling for a Learner, you only to overload the \code{marshal_model} and \code{unmarshal_model} +In order to implement marshaling for a Learner, you need to overload the \code{marshal_model} and \code{unmarshal_model} methods for the class of the learner's model and tag the learner with the \code{"marshal"} property. - To make marshaling accessible in an R6-manner, you should also add the public methods \verb{$marshal()}, \verb{$unmarshal()} and the active binding \verb{$marshaled}. To make this as convenient as possible, the functions \code{learner_marshal(learner)}, \code{learner_unmarshal(learner)} and \code{learner_marshaled(learner)} are provided and can be called from within the public methods. You can verify whether you have correctly implemented marshaling by using the internal test helper -\code{expect_marshalable_learner()}. This is also run by \code{expect_learner()} if a task is provided. +\code{expect_marshalable_learner(learner, task)}. This is also run by \code{expect_learner()} if a task is provided. For a concrete example on how to implement marshaling, see \code{\link{LearnerClassifDebug}}. } diff --git a/man/mlr_learners_classif.debug.Rd b/man/mlr_learners_classif.debug.Rd index f88bdd5ef..51b0ba15a 100644 --- a/man/mlr_learners_classif.debug.Rd +++ b/man/mlr_learners_classif.debug.Rd @@ -25,7 +25,7 @@ The following hyperparameters trigger the following actions: \item{warning_train:}{Probability to signal a warning during train.} \item{x:}{Numeric tuning parameter. Has no effect.} \item{iter:}{Integer parameter for testing hotstarting.} -\item{count_marshaling:}{If \code{TRUE}, \code{marshal_model} will increase the \code{marshal_count} by 1. This value is initialized to \code{FALSE}.} +\item{count_marshaling:}{If \code{TRUE}, \code{marshal_model} will increase the \code{marshal_count} by 1 each time it is called. The default is \code{FALSE}.} } Note that segfaults may not be triggered reliably on your operating system. Also note that if they work as intended, they will tear down your R session immediately! diff --git a/man/resample.Rd b/man/resample.Rd index 17d1ad850..7e7974790 100644 --- a/man/resample.Rd +++ b/man/resample.Rd @@ -61,7 +61,9 @@ providing a set with possible values \code{"task"}, \code{"learner"} and Per default, all input objects are cloned.} \item{unmarshal}{\code{\link{Learner}}\cr -Whether to unmarshal the learner.} +Whether to unmarshal learners that were sent back to the main process in marshaled form. +Setting this to \code{TRUE} does not guarantee that all learners that require marshaling are marshaled. +To achieve this, call \verb{$marshal()} on the result object.} } \value{ \link{ResampleResult}. diff --git a/tests/testthat/test_marshal.R b/tests/testthat/test_marshal.R index 4d90be6df..5a6f865e5 100644 --- a/tests/testthat/test_marshal.R +++ b/tests/testthat/test_marshal.R @@ -24,7 +24,7 @@ test_that("marshaling a marshaled object does nothing", { expect_equal(marshal_model(xm), xm) }) -test_that("unmarshaling a unmarshaled object does nothing", { +test_that("unmarshaling an unmarshaled object does nothing", { x = 1 xm = marshal_model(x) expect_equal(unmarshal_model(xm), x) @@ -57,8 +57,6 @@ test_that("marshal count works for LearnerClassifDebug", { learner$marshal()$unmarshal() expect_equal(learner$model$marshal_count, 2) - # TO make the lily learner more realistic (i.e. (un)marshaling leaves the object unchanged) - # the count_marshaling parameter can also be set to FALSE learner2 = lrn("classif.debug", count_marshaling = FALSE) learner2$train(task) expect_true(is.null(learner2$model$marshal_count)) diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index a61f08592..68a57260c 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -157,6 +157,16 @@ test_that("as_resample_result works for result data", { expect_class(rr2, "ResampleResult") }) +test_that("encapsulation triggers marshaling correctly", { + learner1 = lrn("classif.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + learner2 = lrn("classif.debug", count_marshaling = TRUE, encapsulate = c(train = "none")) + task = tsk("iris") + resampling = rsmp("holdout") + rr1 = resample(task, learner1, resampling, store_models = TRUE, unmarshal = FALSE) + expect_true(rr1$learners[[1]]$marshaled) + rr2 = resample(task, learner2, resampling, store_models = TRUE, unmarshal = FALSE) +}) + test_that("parallel execution automatically triggers marshaling", { learner = lrn("classif.debug", count_marshaling = TRUE) task = tsk("iris") @@ -197,7 +207,13 @@ test_that("marshaling works when store_models is FALSE", { resample(task, learner, resampling, store_models = FALSE, unmarshal = TRUE) }) expect_resample_result(rr) - rr$learners[[1]]$model + expect_true(is.null(rr$learners[[1]]$model)) + + rr1 = with_future(future::sequential, { + resample(task, learner, resampling, store_models = FALSE, unmarshal = TRUE) + }) + expect_resample_result(rr1) + expect_true(is.null(rr1$learners[[1]]$model)) }) @@ -244,3 +260,8 @@ test_that("does not unnecessarily clone state", { learner$train(task) expect_resample_result(resample(task, learner, rsmp("holdout"))) }) + +test_that("marshaling does not change class of learner state", { + rr = resample(tsk("iris"), lrn("classif.debug", encapsulate = c(train = "callr")), rsmp("holdout"), store_models = TRUE) + expect_class(rr$learners[[1]]$state, "learner_state") +}) From af67642411f2b2816e7b2f71bddf22560abbeb44 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Mon, 22 Apr 2024 13:56:48 +0200 Subject: [PATCH 41/47] more cleanup --- R/marshal.R | 12 ++++++------ R/worker.R | 33 ++++++++++++++++----------------- man/marshaling.Rd | 12 ++++++------ tests/testthat/test_resample.R | 2 +- 4 files changed, 29 insertions(+), 30 deletions(-) diff --git a/R/marshal.R b/R/marshal.R index cce744931..e83f16b0a 100644 --- a/R/marshal.R +++ b/R/marshal.R @@ -14,19 +14,19 @@ #' Which takes in a model and returns it in marshaled form. #' This means, that the resulting object can be serialized and de-serialzed without loss of information. #' The marshaled object should be a list with named elements `marshaled` and `packages`, where the former contains -#' the marshaled object, and the latter the package that contains the packages requrired to unmarshal. +#' the marshaled object, and the latter the package that contains the packages requirrd to unmarshal. #' Most importantly, this list should contain the package that contains the `unmarshal_model` method. #' The returned object should have the classes of the original object with the suffix `"_marshaled"` appended and the #' root class should be set to `"marshaled"`. -#' Note that it is not guarateed that `is_marshaled_model(marshal_model(x))` returns `TRUE`. -#' This is because the default `marshal_model(x)` returns `x` as-is. #' * the S3 generic `unmarshal_model(model, inplace ...)`. #' Which takes in the marshaled model and returns it in unmarshaled form. -#' The generic takes care that the packages specified during `"marshal"` are loaded, and errs if they are not. +#' The generic takes care that the packages specified during `"marshal"` are loaded, and errs if they are not availabe. #' Calling this function on a marshaled model should reconstruct the original model, i.e. #' `unmarshal_model(marshal_model(x))` should return `x` as is. #' * the function `is_marshaled_model(model)`. #' This (helper) function returns `TRUE` if the model inherits from class `"marshaled"` and `FALSE` otherwise. +#' Note that it is not guarateed that `is_marshaled_model(marshal_model(x))` returns `TRUE`. +#' This is because the default `marshal_model(x)` returns `x` as-is. #' #' For both `marshal_model` and `unmarshal_model`, the `inplace` argument determines whether in-place marshaling #' should be performed. This is especially relevant in the context of references semantics. @@ -39,8 +39,8 @@ #' methods for the class of the learner's model and tag the learner with the `"marshal"` property. #' To make marshaling accessible in an R6-manner, you should also add the public methods `$marshal()`, `$unmarshal()` #' and the active binding `$marshaled`. -#' To make this as convenient as possible, the functions `learner_marshal(learner)`, `learner_unmarshal(learner)` -#' and `learner_marshaled(learner)` are provided and can be called from within the public methods. +#' To make this as convenient as possible, the functions `learner_marshal(.learner, ...)`, `learner_unmarshal(.learner, ...)` +#' and `learner_marshaled(.learner)` are provided and can be called from the public methods. #' #' You can verify whether you have correctly implemented marshaling by using the internal test helper #' `expect_marshalable_learner(learner, task)`. This is also run by `expect_learner()` if a task is provided. diff --git a/R/worker.R b/R/worker.R index a8f8bdbfb..db5db2046 100644 --- a/R/worker.R +++ b/R/worker.R @@ -259,15 +259,11 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, learner_hash = learner$hash learner = learner_train(learner, task, sets[["train"]], sets[["test"]], mode = mode) - # The model is in marshaled form in case we did callr encapsulation and the model needed marshaling. - # For the predict step we need it unmarshaled. - # If the model is later sent back to main process (when parallelizing) and needs marshaling and the model is already - # in marshaled form, we preserve the marshaled model. - # This means, we temporarily keep both the marshaled and unmarshaled model in RAM - model_marshaled_or_null = get_marshaled_if_needed( + # keep a copy of the marshaled model if it is available and needed later + # this also unmarshals the model if necessary so it can be used for prediction + model_marshaled_or_null = keep_marshaled_if_needed( learner = learner, store_models = store_models, is_sequential = is_sequential, unmarshal = unmarshal ) - learner$model = unmarshal_model(model = learner$model, inplace = FALSE) # predict for each set sets = sets[learner$predict_sets] @@ -277,11 +273,9 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, }, set = names(sets), row_ids = sets) pdatas = discard(pdatas, is.null) - # In case store_models is FALSE, we now discard the model - # Otherwise, we set the model to the marshaled model, if callr encapsulation was used and we are parallelizing - # If callr encapsulation was not used, we still have to marshal the model + # set the model slot after prediction so it can be sent back to the main process set_model_after_predict( - learner = learner, store_models = store_models, is_sequential = is_sequential, model = model_marshaled_or_null + learner = learner, store_models = store_models, is_sequential = is_sequential, model_marshaled = model_marshaled_or_null ) learner_state = set_class(learner$state, c("learner_state", "list")) @@ -289,8 +283,8 @@ workhorse = function(iteration, task, learner, resampling, param_values = NULL, list(learner_state = learner_state, prediction = pdatas, param_values = learner$param_set$values, learner_hash = learner_hash) } -get_marshaled_if_needed = function(learner, store_models, is_sequential, unmarshal) { - if (!is_marshaled_model(learner$model)) { +keep_marshaled_if_needed = function(learner, store_models, is_sequential, unmarshal) { + model_marshaled = if (!is_marshaled_model(learner$model)) { NULL } else if (!store_models) { # else condition means model was marshaled # not storing models, so no need to keep the marhshaled model @@ -304,19 +298,24 @@ get_marshaled_if_needed = function(learner, store_models, is_sequential, unmarsh # keep it, as we have already computed it learner$model } else { # parallel execution - # here, we just need to send back the marshaled model + # here, we need to send back the marshaled model learner$model } + + # need to unmarshal for prediction, does nothing if the model is not marshaled + # we can do it in-place if the marshaled model is not kept + learner$model = unmarshal_model(model = learner$model, inplace = is.null(model_marshaled)) + return(model_marshaled) } -set_model_after_predict = function(learner, store_models, is_sequential, model) { +set_model_after_predict = function(learner, store_models, is_sequential, model_marshaled) { if (!store_models) { lg$debug("Erasing stored model for learner '%s'", learner$id) learner$state$model = NULL - } else if (!is.null(model)) { + } else if (!is.null(model_marshaled)) { # callr + parallelization # or callr + sequential, but unmarshal was FALSE - learner$model = model + learner$model = model_marshaled } else if (!is_sequential) { # parallelization without callr learner$model = marshal_model(learner$model, inplace = TRUE) diff --git a/man/marshaling.Rd b/man/marshaling.Rd index 5843e7525..d37782438 100644 --- a/man/marshaling.Rd +++ b/man/marshaling.Rd @@ -42,19 +42,19 @@ The central functions (and the only methods that are used by \code{mlr3} interna Which takes in a model and returns it in marshaled form. This means, that the resulting object can be serialized and de-serialzed without loss of information. The marshaled object should be a list with named elements \code{marshaled} and \code{packages}, where the former contains -the marshaled object, and the latter the package that contains the packages requrired to unmarshal. +the marshaled object, and the latter the package that contains the packages requirrd to unmarshal. Most importantly, this list should contain the package that contains the \code{unmarshal_model} method. The returned object should have the classes of the original object with the suffix \code{"_marshaled"} appended and the root class should be set to \code{"marshaled"}. -Note that it is not guarateed that \code{is_marshaled_model(marshal_model(x))} returns \code{TRUE}. -This is because the default \code{marshal_model(x)} returns \code{x} as-is. \item the S3 generic \verb{unmarshal_model(model, inplace ...)}. Which takes in the marshaled model and returns it in unmarshaled form. -The generic takes care that the packages specified during \code{"marshal"} are loaded, and errs if they are not. +The generic takes care that the packages specified during \code{"marshal"} are loaded, and errs if they are not availabe. Calling this function on a marshaled model should reconstruct the original model, i.e. \code{unmarshal_model(marshal_model(x))} should return \code{x} as is. \item the function \code{is_marshaled_model(model)}. This (helper) function returns \code{TRUE} if the model inherits from class \code{"marshaled"} and \code{FALSE} otherwise. +Note that it is not guarateed that \code{is_marshaled_model(marshal_model(x))} returns \code{TRUE}. +This is because the default \code{marshal_model(x)} returns \code{x} as-is. } For both \code{marshal_model} and \code{unmarshal_model}, the \code{inplace} argument determines whether in-place marshaling @@ -69,8 +69,8 @@ In order to implement marshaling for a Learner, you need to overload the \code{m methods for the class of the learner's model and tag the learner with the \code{"marshal"} property. To make marshaling accessible in an R6-manner, you should also add the public methods \verb{$marshal()}, \verb{$unmarshal()} and the active binding \verb{$marshaled}. -To make this as convenient as possible, the functions \code{learner_marshal(learner)}, \code{learner_unmarshal(learner)} -and \code{learner_marshaled(learner)} are provided and can be called from within the public methods. +To make this as convenient as possible, the functions \code{learner_marshal(.learner, ...)}, \code{learner_unmarshal(.learner, ...)} +and \code{learner_marshaled(.learner)} are provided and can be called from the public methods. You can verify whether you have correctly implemented marshaling by using the internal test helper \code{expect_marshalable_learner(learner, task)}. This is also run by \code{expect_learner()} if a task is provided. diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index 68a57260c..b3a961639 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -261,7 +261,7 @@ test_that("does not unnecessarily clone state", { expect_resample_result(resample(task, learner, rsmp("holdout"))) }) -test_that("marshaling does not change class of learner state", { +test_that("marshaling does not change class of learner state when reassembling", { rr = resample(tsk("iris"), lrn("classif.debug", encapsulate = c(train = "callr")), rsmp("holdout"), store_models = TRUE) expect_class(rr$learners[[1]]$state, "learner_state") }) From 0e82181dc0018d29769961c3a7bcd7c9a3cbcb46 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Mon, 22 Apr 2024 14:02:04 +0200 Subject: [PATCH 42/47] add test --- tests/testthat/test_resample.R | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index b3a961639..3a35f841f 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -265,3 +265,9 @@ test_that("marshaling does not change class of learner state when reassembling", rr = resample(tsk("iris"), lrn("classif.debug", encapsulate = c(train = "callr")), rsmp("holdout"), store_models = TRUE) expect_class(rr$learners[[1]]$state, "learner_state") }) + +test_that("marshaled model is sent back, when unmarshal is FALSE, sequential exec and callr", { + learner = lrn("classif.debug", count_marshaling = TRUE, encapsulate = c(train = "callr")) + rr = resample(tsk("iris"), learner, rsmp("holdout"), store_models = TRUE, unmarshal = FALSE) + expect_true(rr$learners[[1L]]$marshaled) +}) From 6e87384d7c72d391e809ec4b232ed8821924dc56 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Mon, 22 Apr 2024 14:29:03 +0200 Subject: [PATCH 43/47] more cleanup --- R/worker.R | 2 ++ man-roxygen/param_unmarshal.R | 7 ++++--- man/benchmark.Rd | 7 ++++--- man/resample.Rd | 7 ++++--- tests/testthat/test_resample.R | 1 + 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/R/worker.R b/R/worker.R index db5db2046..344d4a270 100644 --- a/R/worker.R +++ b/R/worker.R @@ -315,9 +315,11 @@ set_model_after_predict = function(learner, store_models, is_sequential, model_m } else if (!is.null(model_marshaled)) { # callr + parallelization # or callr + sequential, but unmarshal was FALSE + # i.e. those cases, where keep_marshaled_if_needed returned the marshaled model learner$model = model_marshaled } else if (!is_sequential) { # parallelization without callr + # in this case we don't have computed the marshaled model yet, so we do it now learner$model = marshal_model(learner$model, inplace = TRUE) } } diff --git a/man-roxygen/param_unmarshal.R b/man-roxygen/param_unmarshal.R index 1776bf97d..ee7727cab 100644 --- a/man-roxygen/param_unmarshal.R +++ b/man-roxygen/param_unmarshal.R @@ -1,4 +1,5 @@ #' @param unmarshal [`Learner`]\cr -#' Whether to unmarshal learners that were sent back to the main process in marshaled form. -#' Setting this to `TRUE` does not guarantee that all learners that require marshaling are marshaled. -#' To achieve this, call `$marshal()` on the result object. +#' Whether to unmarshal learners that were marshaled during the execution. +#' Setting this to `FALSE` does not guarantee that the learners are marshaled. +#' For example, with sequential execution and no encapsulation, marshaling is not necessary. +#' If you want all learners in marshaled form, you need to call `$marshal()` on the result object. diff --git a/man/benchmark.Rd b/man/benchmark.Rd index 064f01094..fa9342bbc 100644 --- a/man/benchmark.Rd +++ b/man/benchmark.Rd @@ -60,9 +60,10 @@ providing a set with possible values \code{"task"}, \code{"learner"} and Per default, all input objects are cloned.} \item{unmarshal}{\code{\link{Learner}}\cr -Whether to unmarshal learners that were sent back to the main process in marshaled form. -Setting this to \code{TRUE} does not guarantee that all learners that require marshaling are marshaled. -To achieve this, call \verb{$marshal()} on the result object.} +Whether to unmarshal learners that were marshaled during the execution. +Setting this to \code{FALSE} does not guarantee that the learners are marshaled. +For example, with sequential execution and no encapsulation, marshaling is not necessary. +If you want all learners in marshaled form, you need to call \verb{$marshal()} on the result object.} } \value{ \link{BenchmarkResult}. diff --git a/man/resample.Rd b/man/resample.Rd index 7e7974790..3bf169748 100644 --- a/man/resample.Rd +++ b/man/resample.Rd @@ -61,9 +61,10 @@ providing a set with possible values \code{"task"}, \code{"learner"} and Per default, all input objects are cloned.} \item{unmarshal}{\code{\link{Learner}}\cr -Whether to unmarshal learners that were sent back to the main process in marshaled form. -Setting this to \code{TRUE} does not guarantee that all learners that require marshaling are marshaled. -To achieve this, call \verb{$marshal()} on the result object.} +Whether to unmarshal learners that were marshaled during the execution. +Setting this to \code{FALSE} does not guarantee that the learners are marshaled. +For example, with sequential execution and no encapsulation, marshaling is not necessary. +If you want all learners in marshaled form, you need to call \verb{$marshal()} on the result object.} } \value{ \link{ResampleResult}. diff --git a/tests/testthat/test_resample.R b/tests/testthat/test_resample.R index 3a35f841f..f0c599af9 100644 --- a/tests/testthat/test_resample.R +++ b/tests/testthat/test_resample.R @@ -165,6 +165,7 @@ test_that("encapsulation triggers marshaling correctly", { rr1 = resample(task, learner1, resampling, store_models = TRUE, unmarshal = FALSE) expect_true(rr1$learners[[1]]$marshaled) rr2 = resample(task, learner2, resampling, store_models = TRUE, unmarshal = FALSE) + expect_false(rr2$learners[[1]]$marshaled) }) test_that("parallel execution automatically triggers marshaling", { From ef59a0b612bb913c7ee130d3d56f5622c61665c7 Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Mon, 22 Apr 2024 14:32:01 +0200 Subject: [PATCH 44/47] better docs --- man-roxygen/param_unmarshal.R | 2 +- man/benchmark.Rd | 2 +- man/resample.Rd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/man-roxygen/param_unmarshal.R b/man-roxygen/param_unmarshal.R index ee7727cab..7fe54639c 100644 --- a/man-roxygen/param_unmarshal.R +++ b/man-roxygen/param_unmarshal.R @@ -2,4 +2,4 @@ #' Whether to unmarshal learners that were marshaled during the execution. #' Setting this to `FALSE` does not guarantee that the learners are marshaled. #' For example, with sequential execution and no encapsulation, marshaling is not necessary. -#' If you want all learners in marshaled form, you need to call `$marshal()` on the result object. +#' If you want to ensure that all learners are in marshaled form, you need to call `$marshal()` on the result object. diff --git a/man/benchmark.Rd b/man/benchmark.Rd index fa9342bbc..92da2fb83 100644 --- a/man/benchmark.Rd +++ b/man/benchmark.Rd @@ -63,7 +63,7 @@ Per default, all input objects are cloned.} Whether to unmarshal learners that were marshaled during the execution. Setting this to \code{FALSE} does not guarantee that the learners are marshaled. For example, with sequential execution and no encapsulation, marshaling is not necessary. -If you want all learners in marshaled form, you need to call \verb{$marshal()} on the result object.} +If you want to ensure that all learners are in marshaled form, you need to call \verb{$marshal()} on the result object.} } \value{ \link{BenchmarkResult}. diff --git a/man/resample.Rd b/man/resample.Rd index 3bf169748..3c298e47c 100644 --- a/man/resample.Rd +++ b/man/resample.Rd @@ -64,7 +64,7 @@ Per default, all input objects are cloned.} Whether to unmarshal learners that were marshaled during the execution. Setting this to \code{FALSE} does not guarantee that the learners are marshaled. For example, with sequential execution and no encapsulation, marshaling is not necessary. -If you want all learners in marshaled form, you need to call \verb{$marshal()} on the result object.} +If you want to ensure that all learners are in marshaled form, you need to call \verb{$marshal()} on the result object.} } \value{ \link{ResampleResult}. From 3707f8c90e302f6b6f29284cda0d7f894a0428fe Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Mon, 22 Apr 2024 16:01:36 +0200 Subject: [PATCH 45/47] fix tests for at and glrn --- inst/testthat/helper_expectations.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index eb7079198..87f4f9df1 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -375,7 +375,8 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { info = sprintf("All hyperparameters of learner %s must be tagged with 'train' or 'predict'. Missing tags for: %s", lrn$id, paste0(names(tags), collapse = ", ")) ) - if ("marshal" %in% lrn$properties) { + # FIXME: remove at and glrn when they have new releases supporting marshaling + if ("marshal" %in% lrn$properties && !test_class(lrn, "GraphLearner") && !test_class(lrn, "AutoTuner")) { assert_function(lrn$marshal) assert_function(lrn$unmarshal) } @@ -385,7 +386,8 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { checkmate::expect_subset(lrn$properties, mlr3::mlr_reflections$learner_properties[[task$task_type]]) testthat::expect_identical(lrn$task_type, task$task_type) - if ("marshal" %in% lrn$properties) { + # FIXME: remove at and glrn when they have new releases supporting marshaling + if ("marshal" %in% lrn$properties && !test_class(lrn, "GraphLearner") && !test_class(lrn, "AutoTuner")) { expect_marshalable_learner(lrn, task) } } From cbf2700f3d3fe8549a8da20c273ee77414380a6a Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Mon, 22 Apr 2024 17:03:19 +0200 Subject: [PATCH 46/47] skip some tests until new versions are released --- inst/testthat/helper_expectations.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index 87f4f9df1..f7f19ee5b 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -376,7 +376,7 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { ) # FIXME: remove at and glrn when they have new releases supporting marshaling - if ("marshal" %in% lrn$properties && !test_class(lrn, "GraphLearner") && !test_class(lrn, "AutoTuner")) { + if ("marshal" %in% lrn$properties && !test_class(lrn, "GraphLearner") && !test_class(lrn, "AutoTuner") && !test_class(lrn, "AutoFSelector")) { assert_function(lrn$marshal) assert_function(lrn$unmarshal) } @@ -387,7 +387,7 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { testthat::expect_identical(lrn$task_type, task$task_type) # FIXME: remove at and glrn when they have new releases supporting marshaling - if ("marshal" %in% lrn$properties && !test_class(lrn, "GraphLearner") && !test_class(lrn, "AutoTuner")) { + if ("marshal" %in% lrn$properties && !test_class(lrn, "GraphLearner") && !test_class(lrn, "AutoTuner") && !test_class(lrn, "AutoFSelector")) { expect_marshalable_learner(lrn, task) } } From c497a7dc177c3eb3b5f53aba973b2d2757c6531c Mon Sep 17 00:00:00 2001 From: Sebastian Fischer Date: Tue, 23 Apr 2024 09:18:59 +0200 Subject: [PATCH 47/47] fix test helpers --- inst/testthat/helper_expectations.R | 47 ++++++++++++++--------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/inst/testthat/helper_expectations.R b/inst/testthat/helper_expectations.R index f7f19ee5b..57b603253 100644 --- a/inst/testthat/helper_expectations.R +++ b/inst/testthat/helper_expectations.R @@ -376,9 +376,9 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { ) # FIXME: remove at and glrn when they have new releases supporting marshaling - if ("marshal" %in% lrn$properties && !test_class(lrn, "GraphLearner") && !test_class(lrn, "AutoTuner") && !test_class(lrn, "AutoFSelector")) { - assert_function(lrn$marshal) - assert_function(lrn$unmarshal) + if ("marshal" %in% lrn$properties && !inherits(lrn, "GraphLearner") && !inherits(lrn, "AutoTuner") && !inherits(lrn, "AutoFSelector")) { + checkmate::expect_function(lrn$marshal) + checkmate::expect_function(lrn$unmarshal) } if (!is.null(task)) { @@ -387,7 +387,7 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { testthat::expect_identical(lrn$task_type, task$task_type) # FIXME: remove at and glrn when they have new releases supporting marshaling - if ("marshal" %in% lrn$properties && !test_class(lrn, "GraphLearner") && !test_class(lrn, "AutoTuner") && !test_class(lrn, "AutoFSelector")) { + if ("marshal" %in% lrn$properties && !inherits(lrn, "GraphLearner") && !inherits(lrn, "AutoTuner") && !inherits(lrn, "AutoFSelector")) { expect_marshalable_learner(lrn, task) } } @@ -399,48 +399,47 @@ expect_learner = function(lrn, task = NULL, check_man = TRUE) { } expect_marshalable_learner = function(learner, task) { - expect_true("marshal" %in% learner$properties) + testthat::expect_true("marshal" %in% learner$properties) learner$state = NULL has_public = function(learner, x) { exists(x, learner, inherits = FALSE) } - expect_true(has_public(learner, "marshal") && test_function(learner$marshal, nargs = 0)) - expect_true(has_public(learner, "unmarshal") && test_function(learner$unmarshal, nargs = 0)) - expect_true(has_public(learner, "marshaled")) + testthat::expect_true(has_public(learner, "marshal") && checkmate::test_function(learner$marshal, nargs = 0)) + testthat::expect_true(has_public(learner, "unmarshal") && checkmate::test_function(learner$unmarshal, nargs = 0)) + testthat::expect_true(has_public(learner, "marshaled")) # (un)marshal only possible after training - expect_error(learner$marshal(), "has not been trained") - expect_error(learner$unmarshal(), "has not been trained") - expect_error(learner$marshaled, "has not been trained") + testthat::expect_error(learner$marshal(), "has not been trained") + testthat::expect_error(learner$unmarshal(), "has not been trained") + testthat::expect_error(learner$marshaled, "has not been trained") learner$train(task) model = learner$model class_prev = class(model) - expect_false(learner$marshaled) - expect_equal(is_marshaled_model(learner$model), learner$marshaled) - expect_invisible(learner$marshal()) - if (!test_class(learner, "GraphLearner")) { - expect_true(learner$marshaled) + testthat::expect_false(learner$marshaled) + testthat::expect_equal(mlr3::is_marshaled_model(learner$model), learner$marshaled) + testthat::expect_invisible(learner$marshal()) + if (!inherits(learner, "GraphLearner")) { + testthat::expect_true(learner$marshaled) } - expect_equal(is_marshaled_model(learner$model), learner$marshaled) + testthat::expect_equal(mlr3::is_marshaled_model(learner$model), learner$marshaled) if (learner$marshaled) { # cannot predict with marshaled learner - expect_error(learner$predict(task), "has not been unmarshaled") + testthat::expect_error(learner$predict(task), "has not been unmarshaled") } # unmarshaling works - expect_invisible(learner$unmarshal()) + testthat::expect_invisible(learner$unmarshal()) # can predict after unmarshaling expect_prediction(learner$predict(task)) # model is reset - expect_equal(learner$model, model) + testthat::expect_equal(learner$model, model) # marshaled is set accordingly - expect_false(learner$marshaled) - - expect_equal(class(learner$model), class_prev) + testthat::expect_false(learner$marshaled) + testthat::expect_equal(class(learner$model), class_prev) } expect_resampling = function(r, task = NULL) { @@ -652,7 +651,7 @@ expect_benchmark_result = function(bmr) { } tab = bmr$aggregate(measures = measures, params = TRUE) - checkmate::assert_list(tab$params) + checkmate::expect_list(tab$params) uhashes = bmr$uhashes expect_uhash(uhashes, len = length(mlr3misc::get_private(bmr)$.data$uhashes()))