Benchmark/rf use case #294
Conversation
…t. still need to incorporate the step logging
| library(data.table) | ||
| library(tidytable) | ||
|
|
||
| cc18_collection = ocl(99) |
There was a problem hiding this comment.
| cc18_collection = ocl(99) | |
| options(mlr3oml.cache = TRUE) | |
| cc18_collection = ocl(99) |
There was a problem hiding this comment.
You can also add this to your .Rprofile
|
|
||
| library(here) | ||
|
|
||
| # define the tasks |
There was a problem hiding this comment.
tsk("oml", task_id = 1067)
similarly hard-code for every task
this will ignore the OpenML resampling
| # define the learners | ||
| mlp = lrn("classif.mlp", | ||
| activation = nn_relu, | ||
| neurons = to_tune( |
There was a problem hiding this comment.
| neurons = to_tune( | |
| neurons = to_tune(ps(n_layers = p_int(lower = 1, upper = 10), latent = p_int(10, 500), .extra_trafo = function(x, param_set) { | |
| list(neurons = rep(x$latent, x$n_layers)) | |
| })) |
There was a problem hiding this comment.
I think this won't work because the libraries don't allow parameter transformations. When I try to run the experiment I get this error:
Error: Inner tuning and parameter transformations are currently not supported.
There was a problem hiding this comment.
My solution for now:
n_layers_values <- 1:10
latent_dim_values <- seq(10, 500, by = 10)
neurons_search_space <- mapply(
neurons,
expand.grid(n_layers = n_layers_values, latent_dim = latent_dim_values)$n_layers,
expand.grid(n_layers = n_layers_values, latent_dim = latent_dim_values)$latent_dim,
SIMPLIFY = FALSE
)
mlp = lrn("classif.mlp",
activation = nn_relu,
# neurons = to_tune(ps(
# n_layers = p_int(lower = 1, upper = 10), latent = p_int(10, 500),
# .extra_trafo = function(x, param_set) {
# list(neurons = rep(x$latent, x$n_layers))
# })
# ),
neurons = to_tune(neurons_search_space)
| c(10, 10), c(10, 20), c(20, 10), c(20, 20) | ||
| ) | ||
| ), | ||
| batch_size = to_tune(16, 32, 64), |
There was a problem hiding this comment.
go bigger
| batch_size = to_tune(16, 32, 64), | |
| batch_size = to_tune(16, 32, 64, 128, 256), |
| tuner = tnr("grid_search"), | ||
| resampling = rsmp("cv"), | ||
| measure = msr("classif.acc"), | ||
| term_evals = 10 |
There was a problem hiding this comment.
likely need more than 10
| term_evals = 10 | |
| term_evals = 100 |
There was a problem hiding this comment.
Run on GPU server (but without all cores)
| # define an AutoTuner that wraps the classif.mlp | ||
| at = auto_tuner( | ||
| learner = mlp, | ||
| tuner = tnr("grid_search"), |
There was a problem hiding this comment.
use MBO: more efficient
| ), | ||
| batch_size = to_tune(16, 32, 64), | ||
| p = to_tune(0.1, 0.9), | ||
| epochs = to_tune(upper = 1000L, internal = TRUE), |
There was a problem hiding this comment.
Consider reducing the max number of epochs
|
|
||
| bmrdt = as.data.table(bmr) | ||
|
|
||
| fwrite(bmrdt, here("R", "rf_Use_case", "results", "bmrdt.csv")) |
There was a problem hiding this comment.
typo
| fwrite(bmrdt, here("R", "rf_Use_case", "results", "bmrdt.csv")) | |
| fwrite(bmrdt, here("R", "rf_use_case", "results", "bmrdt.csv")) |
There was a problem hiding this comment.
Run this with 10 evaluations on the GPU server (with not all of the cores) and report how long it takes.
Only parallelize the learners (one thread per learner) using future.
Run this experiment using the github installation (main branch) of mlr3torch. This will properly handle the interop threads
No description provided.