diff --git a/docs/articles/tikz_prob_algo_simple.png b/docs/articles/tikz_prob_algo_simple.png new file mode 100644 index 00000000..c95ee7b1 Binary files /dev/null and b/docs/articles/tikz_prob_algo_simple.png differ diff --git a/docs/articles/tikz_prob_algo_simple.tex b/docs/articles/tikz_prob_algo_simple.tex new file mode 100644 index 00000000..5787900c --- /dev/null +++ b/docs/articles/tikz_prob_algo_simple.tex @@ -0,0 +1,51 @@ +\documentclass[crop,tikz,convert]{standalone} +\usetikzlibrary{shapes,matrix,positioning,chains,arrows,shadows,decorations.pathmorphing,fit,backgrounds} +\begin{document} + +\begin{tikzpicture}[auto] + \tikzstyle{userinput}=[rectangle, drop shadow, draw=black, fill=black!10, thick, minimum width=4cm, align=center] + \tikzstyle{internal}=[rectangle, drop shadow, draw=black, fill=white, thick, minimum width=4cm, rounded corners, align=center] + \tikzstyle{result}=[ellipse, drop shadow, draw=black, fill=white, thick, align=center, minimum width=3cm] + \tikzstyle{line} = [draw, thick, -latex'] + \tikzstyle{sline} = [draw, thick, -latex',decorate, decoration={snake, segment length=2mm,post length=2mm}] + + \matrix [row sep=10mm, column sep=20mm] { + % first row + \node {}; & + \node {}; & + \node [result] (result) {result}; \\ + + % second row + \node [userinput] (static_problem_part) { + static problem part\\\texttt{data} + }; & + \node [userinput] (dynamic_problem_part) { + dynamic problem function\\ + \texttt{fun(data, ...)} + }; & + \node [userinput] (algorithm) { + algorithm function\\ + \texttt{fun(data, instance, ...)} + }; \\ + + % third row + \node {}; & + \node [userinput] (problem_design) { + problem design\\ + (\texttt{addExperiments}) + }; & + \node [userinput] (algorithm_design) { + algorithm design\\ + (\texttt{addExperiments}) + }; \\ + }; + + \draw [sline] (algorithm) to (result) ; + \draw [line] (static_problem_part) to (dynamic_problem_part); + \draw [sline] (dynamic_problem_part) to node {\texttt{instance}} (algorithm) ; + \draw [line] (static_problem_part) to [out=0, in=0, bend left=20] (algorithm); + + \draw [line] (problem_design) to node {\texttt{...}} (dynamic_problem_part); + \draw [line] (algorithm_design) to node {\texttt{...}} (algorithm); +\end{tikzpicture} +\end{document} diff --git a/docs/articles/v11_ExampleExperiment.html b/docs/articles/v11_ExampleExperiment.html index bcee87eb..ad53b07b 100644 --- a/docs/articles/v11_ExampleExperiment.html +++ b/docs/articles/v11_ExampleExperiment.html @@ -110,6 +110,7 @@

## [1] "iris"
getAlgorithmIds()
## [1] "svm"    "forest"
+

The flow to define experiments is summarized in the following figure:

diff --git a/docs/articles/v20_ErrorHandling.html b/docs/articles/v20_ErrorHandling.html index 9e090198..272cf1e3 100644 --- a/docs/articles/v20_ErrorHandling.html +++ b/docs/articles/v20_ErrorHandling.html @@ -106,15 +106,15 @@

Error Handling

## 2: 9 TRUE TRUE Error in (function (value) : Ooops.

If we want to peek into the R log file of a job to see more context for the error we can use showLog() which opens a pager or use getLog() to get the log as character vector:

writeLines(getLog(id = 9))
-
## ### [bt 2017-01-02 16:19:05]: This is batchtools v0.9.1
-## ### [bt 2017-01-02 16:19:05]: Starting calculation of 1 jobs
-## ### [bt 2017-01-02 16:19:05]: Setting working directory to '/tmp'
-## ### [bt 2017-01-02 16:19:05]: Memory measurement disabled
-## ### [bt 2017-01-02 16:19:05]: Starting job [batchtools job.id=9]
+
## ### [bt 2017-01-03 14:40:17]: This is batchtools v0.9.1
+## ### [bt 2017-01-03 14:40:17]: Starting calculation of 1 jobs
+## ### [bt 2017-01-03 14:40:17]: Setting working directory to '/tmp'
+## ### [bt 2017-01-03 14:40:17]: Memory measurement disabled
+## ### [bt 2017-01-03 14:40:17]: Starting job [batchtools job.id=9]
 ## Error in (function (value)  : Ooops.
 ## 
-## ### [bt 2017-01-02 16:19:05]: Job terminated with an exception [batchtools job.id=9]
-## ### [bt 2017-01-02 16:19:05]: Calculation finished!
+## ### [bt 2017-01-03 14:40:17]: Job terminated with an exception [batchtools job.id=9] +## ### [bt 2017-01-03 14:40:17]: Calculation finished!

You can also grep for error or warning messages:

ids = grepLogs(pattern = "ooops", ignore.case = TRUE)
 print(ids)
diff --git a/docs/index.html b/docs/index.html index f130a20c..e3470eed 100644 --- a/docs/index.html +++ b/docs/index.html @@ -108,6 +108,11 @@

flowr supports LSF, Slurm, Torque and Moab and provides a scatter-gather approach to define computational jobs

+
+

+ Contributing to batchtools

+

This R package is licensed under the LGPL-3. If you encounter problems using this software (lack of documentation, misleading or wrong documentation, unexpected behaviour, bugs, …) or just want to suggest features, please open an issue in the issue tracker. Pull requests are welcome and will be included at the discretion of the author. If you have customized a template file for your (larger) computing site, please share it: fork the repository, place your template in inst/templates and send a pull request.

+
diff --git a/docs/reference/batchExport.html b/docs/reference/batchExport.html index 6ff382e5..80368388 100644 --- a/docs/reference/batchExport.html +++ b/docs/reference/batchExport.html @@ -115,13 +115,13 @@

Examp print(exports)
#> Empty data.table (0 rows) of 2 cols: name,uri
#> NULL
# add a job and required exports batchMap(function(x) x^2 + y + z, x = 1:3, reg = tmp)
#> Adding 3 jobs ...
exports = batchExport(export = list(y = 99, z = 1), reg = tmp)
#> Exporting new objects: 'y','z' ...
print(exports)
#> name uri -#> 1: y /scratch/registry7d21dc2ec4a/exports/y.rds -#> 2: z /scratch/registry7d21dc2ec4a/exports/z.rds
+#> 1: y /scratch/registryb1362b27a62/exports/y.rds +#> 2: z /scratch/registryb1362b27a62/exports/z.rds
submitJobs(reg = tmp)
#> Submitting 3 jobs in 3 chunks using cluster functions 'Interactive' ...
waitForJobs(reg = tmp)
#> Syncing 3 files ...
#> [1] TRUE
stopifnot(loadResult(1, reg = tmp) == 101) # Un-export z exports = batchExport(unexport = "z", reg = tmp)
#> Un-exporting exported objects: 'z' ...
print(exports)
#> name uri -#> 1: y /scratch/registry7d21dc2ec4a/exports/y.rds
+#> 1: y /scratch/registryb1362b27a62/exports/y.rds
# example for an expand.grid()-like operation on parameters -tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#> Sourcing configuration file '/home/lang/.config/batchtools/config.R' ...
ids = batchMap(paste, args = CJ(x = letters[1:3], y = 1:3), more.args = list(sep = ""), reg = tmp)
#> Adding 9 jobs ...
getJobPars(reg = tmp)
#> job.id x y +tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#> Sourcing configuration file '/home/lang/.config/batchtools/config.R' ...
ids = batchMap(paste, args = CJ(x = letters[1:3], y = 1:3), reg = tmp)
#> Adding 9 jobs ...
getJobPars(reg = tmp)
#> job.id x y #> 1: 1 a 1 #> 2: 2 a 2 #> 3: 3 a 3 @@ -160,18 +160,7 @@

Examp #> 6: 6 b 3 #> 7: 7 c 1 #> 8: 8 c 2 -#> 9: 9 c 3

testJob(6, reg = tmp)
#> [1] "b3"
-# example on how to deal with bulky objects -tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#> Sourcing configuration file '/home/lang/.config/batchtools/config.R' ...
ids = batchMap(paste, args = CJ(x = letters[1:3], y = 1:3), more.args = list(sep = ""), reg = tmp)
#> Adding 9 jobs ...
getJobPars(reg = tmp)
#> job.id x y -#> 1: 1 a 1 -#> 2: 2 a 2 -#> 3: 3 a 3 -#> 4: 4 b 1 -#> 5: 5 b 2 -#> 6: 6 b 3 -#> 7: 7 c 1 -#> 8: 8 c 2 -#> 9: 9 c 3
testJob(6, reg = tmp)
#> [1] "b3"
+#> 9: 9 c 3
testJob(6, reg = tmp)
#> [1] "b 3"
#> job.id submitted started done -#> 1: 1 2017-01-02 16:18:58 2017-01-02 16:18:58 2017-01-02 16:18:58 -#> 2: 2 2017-01-02 16:18:58 2017-01-02 16:18:58 2017-01-02 16:18:58 -#> 3: 3 2017-01-02 16:18:58 2017-01-02 16:18:58 2017-01-02 16:18:58 +#> 1: 1 2017-01-03 14:40:10 2017-01-03 14:40:10 2017-01-03 14:40:10 +#> 2: 2 2017-01-03 14:40:10 2017-01-03 14:40:10 2017-01-03 14:40:10 +#> 3: 3 2017-01-03 14:40:10 2017-01-03 14:40:10 2017-01-03 14:40:10 #> error memory batch.id #> 1: Error in (function (x) : x must be > 0 NA cfInteractive #> 2: NA NA cfInteractive #> 3: NA NA cfInteractive #> job.hash time.queued time.running pars -#> 1: jobe391714b4f68672032f2e58f85db5fc1 0 secs 0 secs <list> -#> 2: job5f463599c1d071a8bc8a29c625508590 0 secs 0 secs <list> -#> 3: job4cdd09efa2c152107bf683bc55259691 0 secs 0 secs <list> +#> 1: jobcd113a712bba6e03f2b65a5de1c5b081 0 secs 0 secs <list> +#> 2: job3a6d57d123805b6c055767888e18f571 0 secs 0 secs <list> +#> 3: jobc877ea486a1397f76920d4acd3fe35d8 0 secs 0 secs <list> #> resources tags #> 1: <list> tag1 #> 2: <list> tag1,tag2 diff --git a/docs/reference/makeRegistry.html b/docs/reference/makeRegistry.html index 906dd039..5141496c 100644 --- a/docs/reference/makeRegistry.html +++ b/docs/reference/makeRegistry.html @@ -205,7 +205,7 @@

See a

Examples

tmp = makeRegistry(file.dir = NA, make.default = FALSE)
#> Sourcing configuration file '/home/lang/.config/batchtools/config.R' ...
print(tmp)
#> Job Registry #> Name : Interactive -#> File dir: /scratch/registry7d217ec115a +#> File dir: /scratch/registryb13153650b8 #> Work dir: /home/lang/Projekte/batchtools/docs/reference #> Jobs : 0 #> Seed : 396414167
diff --git a/docs/reference/showLog.html b/docs/reference/showLog.html index 5dbee55b..4eb70952 100644 --- a/docs/reference/showLog.html +++ b/docs/reference/showLog.html @@ -127,14 +127,14 @@

Examp #> Done : 4 ( 80.0%) #> Error : 1 ( 20.0%) #> Expired : 0 ( 0.0%)

-writeLines(getLog(ids[1], reg = tmp))
#> ### [bt 2017-01-02 16:19:01]: This is batchtools v0.9.1 -#> ### [bt 2017-01-02 16:19:01]: Starting calculation of 1 jobs -#> ### [bt 2017-01-02 16:19:01]: Setting working directory to '/home/lang/Projekte/batchtools/docs/reference' -#> ### [bt 2017-01-02 16:19:01]: Memory measurement disabled -#> ### [bt 2017-01-02 16:19:01]: Starting job [batchtools job.id=1] +writeLines(getLog(ids[1], reg = tmp))
#> ### [bt 2017-01-03 14:40:13]: This is batchtools v0.9.1 +#> ### [bt 2017-01-03 14:40:13]: Starting calculation of 1 jobs +#> ### [bt 2017-01-03 14:40:13]: Setting working directory to '/home/lang/Projekte/batchtools/docs/reference' +#> ### [bt 2017-01-03 14:40:13]: Memory measurement disabled +#> ### [bt 2017-01-03 14:40:13]: Starting job [batchtools job.id=1] #> -#> ### [bt 2017-01-02 16:19:01]: Job terminated successfully [batchtools job.id=1] -#> ### [bt 2017-01-02 16:19:01]: Calculation finished!
## Not run: ------------------------------------ +#> ### [bt 2017-01-03 14:40:13]: Job terminated successfully [batchtools job.id=1] +#> ### [bt 2017-01-03 14:40:13]: Calculation finished!
## Not run: ------------------------------------ # showLog(ids[1], reg = tmp) ## --------------------------------------------- diff --git a/docs/reference/submitJobs.html b/docs/reference/submitJobs.html index 2b350222..8eeccc0e 100644 --- a/docs/reference/submitJobs.html +++ b/docs/reference/submitJobs.html @@ -173,13 +173,13 @@

Examp submitJobs(ids, resources = res, reg = tmp)

#> Submitting 4 jobs in 4 chunks using cluster functions 'Interactive' ...
# Retrive information about memory, combine with parameters info = ijoin(getJobStatus(reg = tmp)[, .(job.id, memory)], getJobPars(reg = tmp))
#> Syncing 4 files ...
print(info)
#> job.id memory n p -#> 1: 1 51.1 1e+04 10 +#> 1: 1 51.3 1e+04 10 #> 2: 2 63.6 1e+05 10 #> 3: 3 56.3 1e+04 50 #> 4: 4 124.6 1e+05 50
# Combine job info with results -> each job is aggregated using mean() ijoin(info, reduceResultsDataTable(fun = function(res) list(res = mean(res)), reg = tmp))
#> job.id memory n p res -#> 1: 1 51.1 1e+04 10 0.4999599 +#> 1: 1 51.3 1e+04 10 0.4999599 #> 2: 2 63.6 1e+05 10 0.5006067 #> 3: 3 56.3 1e+04 50 0.4998562 #> 4: 4 124.6 1e+05 50 0.4999849
diff --git a/vignettes/tikz_prob_algo_simple.png b/vignettes/tikz_prob_algo_simple.png new file mode 100644 index 00000000..c95ee7b1 Binary files /dev/null and b/vignettes/tikz_prob_algo_simple.png differ diff --git a/vignettes/tikz_prob_algo_simple.svg b/vignettes/tikz_prob_algo_simple.svg deleted file mode 100644 index bd75f9eb..00000000 --- a/vignettes/tikz_prob_algo_simple.svg +++ /dev/null @@ -1,579 +0,0 @@ - - - - diff --git a/vignettes/tikz_prob_algo_simple.tex b/vignettes/tikz_prob_algo_simple.tex index 66fb0bf0..5787900c 100644 --- a/vignettes/tikz_prob_algo_simple.tex +++ b/vignettes/tikz_prob_algo_simple.tex @@ -1,4 +1,4 @@ -\documentclass[crop,tikz,convert={outext=.svg}]{standalone} +\documentclass[crop,tikz,convert]{standalone} \usetikzlibrary{shapes,matrix,positioning,chains,arrows,shadows,decorations.pathmorphing,fit,backgrounds} \begin{document} diff --git a/vignettes/v11_ExampleExperiment.Rmd b/vignettes/v11_ExampleExperiment.Rmd index 12c76b43..7f4b2e4d 100644 --- a/vignettes/v11_ExampleExperiment.Rmd +++ b/vignettes/v11_ExampleExperiment.Rmd @@ -99,7 +99,9 @@ getProblemIds() getAlgorithmIds() ``` The flow to define experiments is summarized in the following figure: -![flowchart](tikz_prob_algo_simple.svg) +```{r, echo=FALSE} +knitr::include_graphics("tikz_prob_algo_simple.png") +``` # Creating jobs