feat: add asynchronous parallelization (#395)

* feat: add rush parallelization

* fix: collate

* fix: hotstart flag

* feat: add rush to ti

* refactor: add freeze_archive option

* refactor: start workers

* fix: start workers

* fix: archive as.data.table

* feat: add callbacks

* refactor: objective tuning

* docs: callbacks

* test: objective tuning

* docs: add missing params

* fix: objective

* docs: add parameter

* feat: add rush measures callback

* fix: pkgdown

* feat: option to start workers when instance is created

* feat: add async grid search

* feat: save log messages from worker

* feat: add debug messages

* test: hotstarting works with rush

* feat: add hotstart threshold

* fix: resample result

* refactor: TuningInstanceRush

* benchmark: add setup

* refactor: remove benchmark

* draft

* tests: pipeline

* feat: log performance

* fix: save default values

* fix: insert values

* chore: leanify

* chore: import rush

* feat: add errors and warnings

* feat: add log messages to archive

* feat: allow to pass rush instance

* feat: add mlflow callback

* feat: add auto tuner rush

* feat: add rush

* feat: as_tuner

* refactor: async

* draft

* draft

* draft

* draft

* draft

* tests: skip async tests

* draft

* chore: pkgdown

* fix: description

* ci: remove as-cran flag

* chore: update news

* chore: news

.github/workflows/dev-cmd-check.yml

@@ -35,6 +35,10 @@ jobs:
         with:
           r-version: ${{ matrix.config.r }}
 
+      - uses: supercharge/[email protected]
+        with:
+          redis-version: 7
+
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
           extra-packages: any::rcmdcheck
@@ -45,3 +49,5 @@ jobs:
         shell: Rscript {0}
 
       - uses: r-lib/actions/check-r-package@v2
+        with:
+          args: 'c("--no-manual")' # "--as-cran" prevents to start external processes

.github/workflows/r-cmd-check.yml

@@ -34,9 +34,15 @@ jobs:
         with:
           r-version: ${{ matrix.config.r }}
 
+      - uses: supercharge/[email protected]
+        with:
+          redis-version: 7
+
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
           extra-packages: any::rcmdcheck
           needs: check
 
       - uses: r-lib/actions/check-r-package@v2
+        with:
+          args: 'c("--no-manual")' # "--as-cran" prevents to start external processes

.gitignore

@@ -182,5 +182,4 @@ revdep/
 # misc
 Meta/
 attic/
-inst/doc
-benchmark/
+inst/docd

DESCRIPTION

@@ -29,49 +29,70 @@ Depends:
     paradox (>= 0.10.0),
     R (>= 3.1.0)
 Imports:
-    bbotk (>= 0.7.3),
+    bbotk (>= 0.8.0.9000),
     checkmate (>= 2.0.0),
     data.table,
     lgr,
     mlr3misc (>= 0.13.0),
+    rush,
     R6
 Suggests:
     adagio,
+    future,
     GenSA,
     irace,
     knitr,
+    mlflow,
     mlr3learners (>= 0.5.5),
     mlr3pipelines,
     nloptr,
+    redux,
     rmarkdown,
     rpart,
     testthat (>= 3.0.0),
     xgboost
+Remotes:
+    mlr-org/bbotk,
+    mlr-org/rush
 Config/testthat/edition: 3
-Config/testthat/parallel: true
+Config/testthat/parallel: false
 Encoding: UTF-8
 NeedsCompilation: no
 Roxygen: list(markdown = TRUE)
 RoxygenNote: 7.3.1
 Collate:
-    'ArchiveTuning.R'
+    'ArchiveAsyncTuning.R'
+    'ArchiveBatchTuning.R'
     'AutoTuner.R'
-    'CallbackTuning.R'
-    'ContextEval.R'
+    'CallbackAsyncTuning.R'
+    'CallbackBatchTuning.R'
+    'ContextAsyncTuning.R'
+    'ContextBatchTuning.R'
     'ObjectiveTuning.R'
+    'ObjectiveTuningAsync.R'
+    'ObjectiveTuningBatch.R'
     'mlr_tuners.R'
     'Tuner.R'
-    'TunerCmaes.R'
-    'TunerDesignPoints.R'
-    'TunerFromOptimizer.R'
-    'TunerGenSA.R'
-    'TunerGridSearch.R'
-    'TunerIrace.R'
-    'TunerNLoptr.R'
-    'TunerRandomSearch.R'
-    'TuningInstanceSingleCrit.R'
-    'TuningInstanceMulticrit.R'
+    'TunerAsync.R'
+    'TunerAsyncDesignPoints.R'
+    'TunerAsyncFromOptimizerAsync.R'
+    'TunerAsyncGridSearch.R'
+    'TunerAsyncRandomSearch.R'
+    'TunerBatch.R'
+    'TunerBatchCmaes.R'
+    'TunerBatchDesignPoints.R'
+    'TunerBatchFromBatchOptimizer.R'
+    'TunerBatchGenSA.R'
+    'TunerBatchGridSearch.R'
+    'TunerBatchIrace.R'
+    'TunerBatchNLoptr.R'
+    'TunerBatchRandomSearch.R'
+    'TuningInstanceBatchSingleCrit.R'
+    'TuningInstanceAsyncMulticrit.R'
+    'TuningInstanceAsyncSingleCrit.R'
+    'TuningInstanceBatchMulticrit.R'
     'as_search_space.R'
+    'as_tuner.R'
     'assertions.R'
     'auto_tuner.R'
     'bibentries.R'

NAMESPACE

@@ -1,31 +1,59 @@
 # Generated by roxygen2: do not edit by hand
 
-S3method(as.data.table,ArchiveTuning)
+S3method(as.data.table,ArchiveAsyncTuning)
+S3method(as.data.table,ArchiveBatchTuning)
 S3method(as.data.table,DictionaryTuner)
 S3method(as_search_space,Learner)
 S3method(as_search_space,ParamSet)
+S3method(as_tuner,Tuner)
+S3method(as_tuners,default)
+S3method(as_tuners,list)
 S3method(extract_inner_tuning_archives,BenchmarkResult)
 S3method(extract_inner_tuning_archives,ResampleResult)
 S3method(extract_inner_tuning_results,BenchmarkResult)
 S3method(extract_inner_tuning_results,ResampleResult)
-export(ArchiveTuning)
+export(ArchiveAsyncTuning)
+export(ArchiveBatchTuning)
 export(AutoTuner)
-export(ContextEval)
+export(CallbackAsyncTuning)
+export(CallbackBatchTuning)
+export(ContextAsyncTuning)
+export(ContextBatchTuning)
 export(ObjectiveTuning)
+export(ObjectiveTuningAsync)
+export(ObjectiveTuningBatch)
 export(Tuner)
-export(TunerCmaes)
-export(TunerDesignPoints)
-export(TunerFromOptimizer)
-export(TunerGenSA)
-export(TunerGridSearch)
-export(TunerIrace)
-export(TunerNLoptr)
-export(TunerRandomSearch)
-export(TuningInstanceMultiCrit)
-export(TuningInstanceSingleCrit)
+export(TunerAsync)
+export(TunerAsyncDesignPoints)
+export(TunerAsyncFromOptimizerAsync)
+export(TunerAsyncGridSearch)
+export(TunerAsyncRandomSearch)
+export(TunerBatch)
+export(TunerBatchCmaes)
+export(TunerBatchDesignPoints)
+export(TunerBatchFromOptimizerBatch)
+export(TunerBatchGenSA)
+export(TunerBatchGridSearch)
+export(TunerBatchIrace)
+export(TunerBatchNLoptr)
+export(TunerBatchRandomSearch)
+export(TuningInstanceAsyncMultiCrit)
+export(TuningInstanceAsyncSingleCrit)
+export(TuningInstanceBatchMultiCrit)
+export(TuningInstanceBatchSingleCrit)
 export(as_search_space)
+export(as_tuner)
+export(as_tuners)
+export(assert_tuner)
+export(assert_tuner_async)
+export(assert_tuner_batch)
+export(assert_tuners)
+export(assert_tuning_instance)
+export(assert_tuning_instance_async)
+export(assert_tuning_instance_batch)
 export(auto_tuner)
-export(callback_tuning)
+export(callback_async_tuning)
+export(callback_batch_tuning)
 export(clbk)
 export(clbks)
 export(extract_inner_tuning_archives)
@@ -34,6 +62,7 @@ export(mlr_callbacks)
 export(mlr_terminators)
 export(mlr_tuners)
 export(ti)
+export(ti_async)
 export(tnr)
 export(tnrs)
 export(trm)
@@ -46,6 +75,7 @@ import(data.table)
 import(mlr3)
 import(mlr3misc)
 import(paradox)
+import(rush)
 importFrom(R6,R6Class)
 importFrom(bbotk,mlr_terminators)
 importFrom(bbotk,trm)

NEWS.md

@@ -1,5 +1,13 @@
 # mlr3tuning (development version)
 
+* feat: Introduce asynchronous optimization with the `TunerAsync` and `TuningInstanceAsync*` classes.
+* BREAKING CHANGE: The `Tuner` class is `TunerBatch` now.
+* BREAKING CHANGE: THe `TuningInstanceSingleCrit` and `TuningInstanceMultiCrit` classes are `TuningInstanceBatchSingleCrit` and `TuningInstanceBatchMultiCrit` now.
+* BREAKING CHANGE: The `CallbackTuning` class is `CallbackBatchTuning` now.
+* BREAKING CHANGE: The `ContextEval` class is `ContextBatchTuning` now.
+* refactor: Remove hotstarting from batch optimization due to low performance.
+* refactor: The option `evaluate_default` is a callback now.
+
 # mlr3tuning 0.20.0
 
 * compatibility: Work with new paradox version 1.0.0
@@ -52,7 +60,7 @@
 
 * feat: Add new callback that enables early stopping while tuning to `mlr_callbacks`.
 * feat: Add new callback that backups the benchmark result to disk after each batch.
-* feat: Create custom callbacks with the `callback_tuning()` function.
+* feat: Create custom callbacks with the `callback_batch_tuning()` function.
 
 # mlr3tuning 0.15.0