refactor: drop assertion of insample performance of surrogate as a feature completely

Author: sumny

R/Surrogate.R

@@ -161,16 +161,6 @@ Surrogate = R6Class("Surrogate",
       }
     },
 
-    #' @field insample_perf (`numeric()`)\cr
-    #'   Surrogate model's current insample performance.
-    insample_perf = function(rhs) {
-      if (missing(rhs)) {
-        private$.insample_perf %??% NaN
-      } else {
-        stop("$insample_perf is read-only.")
-      }
-    },
-
     #' @field param_set ([paradox::ParamSet])\cr
     #'   Set of hyperparameters.
     param_set = function(rhs) {
@@ -181,11 +171,6 @@ Surrogate = R6Class("Surrogate",
       }
     },
 
-    #' @template field_assert_insample_perf_surrogate
-    assert_insample_perf = function(rhs) {
-      stop("Abstract.")
-    },
-
     #' @template field_packages_surrogate
     packages = function(rhs) {
       if (missing(rhs)) {
@@ -231,8 +216,6 @@ Surrogate = R6Class("Surrogate",
 
     .cols_y = NULL,
 
-    .insample_perf = NULL,
-
     .param_set = NULL,
 
     .update = function() {

R/SurrogateLearner.R

@@ -5,22 +5,6 @@
 #'
 #' @section Parameters:
 #' \describe{
-#' \item{`assert_insample_perf`}{`logical(1)`\cr
-#'   Should the insample performance of the [mlr3::LearnerRegr] be asserted after updating the surrogate?
-#'   If the assertion fails (i.e., the insample performance based on the `perf_measure` does not meet the
-#'   `perf_threshold`), an error is thrown.
-#'   Default is `FALSE`.
-#' }
-#' \item{`perf_measure`}{[mlr3::MeasureRegr]\cr
-#'   Performance measure which should be use to assert the insample performance of the [mlr3::LearnerRegr].
-#'   Only relevant if `assert_insample_perf = TRUE`.
-#'   Default is [mlr3::mlr_measures_regr.rsq].
-#' }
-#' \item{`perf_threshold`}{`numeric(1)`\cr
-#'   Threshold the insample performance of the [mlr3::LearnerRegr] should be asserted against.
-#'   Only relevant if `assert_insample_perf = TRUE`.
-#'   Default is `0`.
-#' }
 #' \item{`catch_errors`}{`logical(1)`\cr
 #'   Should errors during updating the surrogate be caught and propagated to the `loop_function` which can then handle
 #'   the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation?
@@ -89,15 +73,10 @@ SurrogateLearner = R6Class("SurrogateLearner",
       assert_string(col_y, null.ok = TRUE)
 
       ps = ps(
-        assert_insample_perf = p_lgl(),
-        perf_measure = p_uty(custom_check = function(x) check_r6(x, classes = "MeasureRegr")),  # FIXME: actually want check_measure
-        perf_threshold = p_dbl(lower = -Inf, upper = Inf),
         catch_errors = p_lgl(),
         impute_method = p_fct(c("mean", "random"), default = "random")
       )
-      ps$values = list(assert_insample_perf = FALSE, catch_errors = TRUE, impute_method = "random")
-      ps$add_dep("perf_measure", on = "assert_insample_perf", cond = CondEqual$new(TRUE))
-      ps$add_dep("perf_threshold", on = "assert_insample_perf", cond = CondEqual$new(TRUE))
+      ps$values = list(catch_errors = TRUE, impute_method = "random")
 
       super$initialize(learner = learner, archive = archive, cols_x = cols_x, cols_y = col_y, param_set = ps)
     },
@@ -138,47 +117,6 @@ SurrogateLearner = R6Class("SurrogateLearner",
       1L
     },
 
-    #' @template field_assert_insample_perf_surrogate
-    assert_insample_perf = function(rhs) {
-      if (missing(rhs)) {
-        if (!self$param_set$values$assert_insample_perf) {
-          return(invisible(self$insample_perf))
-        }
-
-        perf_measure = self$param_set$values$perf_measure %??% mlr_measures$get("regr.rsq")
-        perf_threshold = self$param_set$values$perf_threshold %??% 0
-        check = if (perf_measure$minimize) {
-          self$insample_perf < perf_threshold
-        } else {
-          self$insample_perf > perf_threshold
-        }
-
-        if (!check) {
-          stop("Current insample performance of the Surrogate Model does not meet the performance threshold.")
-        }
-        invisible(self$insample_perf)
-      } else {
-        stop("$assert_insample_perf is read-only.")
-      }
-
-      if (!self$param_set$values$assert_insample_perf) {
-        return(invisible(self$insample_perf))
-      }
-
-      perf_measure = self$param_set$values$perf_measure %??% mlr_measures$get("regr.rsq")
-      perf_threshold = self$param_set$values$perf_threshold %??% 0
-      check = if (perf_measure$minimize) {
-        self$insample_perf < perf_threshold
-      } else {
-        self$insample_perf > perf_threshold
-      }
-
-      if (!check) {
-        stop("Current insample performance of the Surrogate Model does not meet the performance threshold.")
-      }
-      invisible(self$insample_perf)
-    },
-
     #' @template field_packages_surrogate
     packages = function(rhs) {
       if (missing(rhs)) {
@@ -218,23 +156,15 @@ SurrogateLearner = R6Class("SurrogateLearner",
 
   private = list(
     # Train learner with new data.
-    # Also calculates the insample performance based on the `perf_measure` hyperparameter if `assert_insample_perf = TRUE`.
     .update = function() {
       xydt = self$archive$data[, c(self$cols_x, self$cols_y), with = FALSE]
       task = TaskRegr$new(id = "surrogate_task", backend = xydt, target = self$cols_y)
       assert_learnable(task, learner = self$learner)
       self$learner$train(task)
-
-      if (self$param_set$values$assert_insample_perf) {
-        measure = assert_measure(self$param_set$values$perf_measure %??% mlr_measures$get("regr.rsq"), task = task, learner = self$learner)
-        private$.insample_perf = self$learner$predict(task)$score(measure, task = task, learner = self$learner)
-        self$assert_insample_perf
-      }
     },
 
     # Train learner with new data.
     # Operates on an asynchronous archive and performs imputation as needed.
-    # Also calculates the insample performance based on the `perf_measure` hyperparameter if `assert_insample_perf = TRUE`.
     .update_async = function() {
       xydt = self$archive$rush$fetch_tasks_with_state(states = c("queued", "running", "finished"))[, c(self$cols_x, self$cols_y, "state"), with = FALSE]
       if (self$param_set$values$impute_method == "mean") {
@@ -250,12 +180,6 @@ SurrogateLearner = R6Class("SurrogateLearner",
       task = TaskRegr$new(id = "surrogate_task", backend = xydt, target = self$cols_y)
       assert_learnable(task, learner = self$learner)
       self$learner$train(task)
-
-      if (self$param_set$values$assert_insample_perf) {
-        measure = assert_measure(self$param_set$values$perf_measure %??% mlr_measures$get("regr.rsq"), task = task, learner = self$learner)
-        private$.insample_perf = self$learner$predict(task)$score(measure, task = task, learner = self$learner)
-        self$assert_insample_perf
-      }
     },
 
     .reset = function() {

R/SurrogateLearnerCollection.R

@@ -7,22 +7,6 @@
 #'
 #' @section Parameters:
 #' \describe{
-#' \item{`assert_insample_perf`}{`logical(1)`\cr
-#'   Should the insample performance of the [mlr3::LearnerRegr] be asserted after updating the surrogate?
-#'   If the assertion fails (i.e., the insample performance based on the `perf_measure` does not meet the
-#'   `perf_threshold`), an error is thrown.
-#'   Default is `FALSE`.
-#' }
-#' \item{`perf_measure`}{List of [mlr3::MeasureRegr]\cr
-#'   Performance measures which should be use to assert the insample performance of the [mlr3::LearnerRegr].
-#'   Only relevant if `assert_insample_perf = TRUE`.
-#'   Default is [mlr3::mlr_measures_regr.rsq] for each learner.
-#' }
-#' \item{`perf_threshold`}{List of `numeric(1)`\cr
-#'   Thresholds the insample performance of the [mlr3::LearnerRegr] should be asserted against.
-#'   Only relevant if `assert_insample_perf = TRUE`.
-#'   Default is `0` for each learner.
-#' }
 #' \item{`catch_errors`}{`logical(1)`\cr
 #'   Should errors during updating the surrogate be caught and propagated to the `loop_function` which can then handle
 #'   the failed acquisition function optimization (as a result of the failed surrogate) appropriately by, e.g., proposing a randomly sampled point for evaluation?
@@ -104,15 +88,10 @@ SurrogateLearnerCollection = R6Class("SurrogateLearnerCollection",
       assert_character(cols_y, len = length(learners), null.ok = TRUE)
 
       ps = ps(
-        assert_insample_perf = p_lgl(),
-        perf_measures = p_uty(custom_check = function(x) check_list(x, types = "MeasureRegr", any.missing = FALSE, len = length(learners))),  # FIXME: actually want check_measures
-        perf_thresholds = p_uty(custom_check = function(x) check_double(x, lower = -Inf, upper = Inf, any.missing = FALSE, len = length(learners))),
         catch_errors = p_lgl(),
         impute_method = p_fct(c("mean", "random"), default = "random")
       )
-      ps$values = list(assert_insample_perf = FALSE, catch_errors = TRUE, impute_method = "random")
-      ps$add_dep("perf_measures", on = "assert_insample_perf", cond = CondEqual$new(TRUE))
-      ps$add_dep("perf_thresholds", on = "assert_insample_perf", cond = CondEqual$new(TRUE))
+      ps$values = list(catch_errors = TRUE, impute_method = "random")
 
       super$initialize(learner = learners, archive = archive, cols_x = cols_x, cols_y = cols_y, param_set = ps)
     },
@@ -159,33 +138,6 @@ SurrogateLearnerCollection = R6Class("SurrogateLearnerCollection",
       length(self$learner)
     },
 
-    #' @template field_assert_insample_perf_surrogate
-    assert_insample_perf = function(rhs) {
-      if (missing(rhs)) {
-        check = all(pmap_lgl(
-          list(
-            insample_perf = self$insample_perf,
-            perf_threshold = self$param_set$values$perf_thresholds %??% rep(0, self$n_learner),
-            perf_measure = self$param_set$values$perf_measures %??% replicate(self$n_learner, mlr_measures$get("regr.rsq"), simplify = FALSE)
-          ),
-          .f = function(insample_perf, perf_threshold, perf_measure) {
-            if (perf_measure$minimize) {
-              insample_perf < perf_threshold
-            } else {
-              insample_perf > perf_threshold
-            }
-          })
-        )
-
-        if (!check) {
-          stop("Current insample performance of the Surrogate Model does not meet the performance threshold.")
-        }
-        invisible(self$insample_perf)
-      } else {
-        stop("$assert_insample_perf is read-only.")
-      }
-    },
-
     #' @template field_packages_surrogate
     packages = function(rhs) {
       if (missing(rhs)) {
@@ -230,7 +182,6 @@ SurrogateLearnerCollection = R6Class("SurrogateLearnerCollection",
   private = list(
 
     # Train learner with new data.
-    # Also calculates the insample performance based on the `perf_measures` hyperparameter if `assert_insample_perf = TRUE`.
     .update = function() {
       assert_true((length(self$cols_y) == length(self$learner)) || length(self$cols_y) == 1L)  # either as many cols_y as learner or only one
       one_to_multiple = length(self$cols_y) == 1L
@@ -255,21 +206,10 @@ SurrogateLearnerCollection = R6Class("SurrogateLearnerCollection",
       } else {
         names(self$learner) = self$cols_y
       }
-
-      if (self$param_set$values$assert_insample_perf) {
-        private$.insample_perf = setNames(pmap_dbl(list(learner = self$learner, task = tasks, perf_measure = self$param_set$values$perf_measures %??% replicate(self$n_learner, mlr_measures$get("regr.rsq"), simplify = FALSE)),
-          .f = function(learner, task, perf_measure) {
-            assert_measure(perf_measure, task = task, learner = learner)
-            learner$predict(task)$score(perf_measure, task = task, learner = learner)
-          }
-        ), nm = map_chr(self$param_set$values$perf_measures, "id"))
-        self$assert_insample_perf
-      }
     },
 
     # Train learner with new data.
     # Operates on an asynchronous archive and performs imputation as needed.
-    # Also calculates the insample performance based on the `perf_measures` hyperparameter if `assert_insample_perf = TRUE`.
     .update_async = function() {
       assert_true((length(self$cols_y) == length(self$learner)) || length(self$cols_y) == 1L)  # either as many cols_y as learner or only one
       one_to_multiple = length(self$cols_y) == 1L
@@ -309,16 +249,6 @@ SurrogateLearnerCollection = R6Class("SurrogateLearnerCollection",
       } else {
         names(self$learner) = self$cols_y
       }
-
-      if (self$param_set$values$assert_insample_perf) {
-        private$.insample_perf = setNames(pmap_dbl(list(learner = self$learner, task = tasks, perf_measure = self$param_set$values$perf_measures %??% replicate(self$n_learner, mlr_measures$get("regr.rsq"), simplify = FALSE)),
-          .f = function(learner, task, perf_measure) {
-            assert_measure(perf_measure, task = task, learner = learner)
-            learner$predict(task)$score(perf_measure, task = task, learner = learner)
-          }
-        ), nm = map_chr(self$param_set$values$perf_measures, "id"))
-        self$assert_insample_perf
-      }
     },
 
     .reset = function() {