From d9957d94a7578c5305ab87cde9174bb78917425d Mon Sep 17 00:00:00 2001 From: Cornelius-G Date: Mon, 4 Sep 2023 17:48:59 +0200 Subject: [PATCH 1/6] Allow to pass Optim.Options to bat_findmode --- ext/BATOptimExt.jl | 9 ++++----- src/extdefs/optim_defs.jl | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ext/BATOptimExt.jl b/ext/BATOptimExt.jl index 708d75770..7b856dd11 100644 --- a/ext/BATOptimExt.jl +++ b/ext/BATOptimExt.jl @@ -31,6 +31,7 @@ BAT.ext_default(::BAT.PackageExtension{:Optim}, ::Val{:DEFAULT_OPTALG}) = Optim. BAT.ext_default(::BAT.PackageExtension{:Optim}, ::Val{:NELDERMEAD_ALG}) = Optim.NelderMead() BAT.ext_default(::BAT.PackageExtension{:Optim}, ::Val{:LBFGS_ALG}) = Optim.LBFGS() +BAT.ext_default(::BAT.PackageExtension{:Optim}, ::Val{:DEFAULT_OPTS}) = Optim.Options(store_trace = true, extended_trace=true) struct NLSolversFG!{F,AD} <: Function f::F @@ -69,7 +70,7 @@ function BAT.bat_findmode_impl(target::AnyMeasureOrDensity, algorithm::OptimAlg, # Maximize density of original target, but run in transformed space, don't apply LADJ: f = fchain(inv_trafo, logdensityof(target), -) - optim_result = _optim_minimize(f, x_init, algorithm.optalg, context) + optim_result = _optim_minimize(f, x_init, algorithm.optalg, algorithm.options, context) r_optim = Optim.MaximizationWrapper(optim_result) transformed_mode = Optim.minimizer(r_optim.res) result_mode = inv_trafo(transformed_mode) @@ -81,18 +82,16 @@ function BAT.bat_findmode_impl(target::AnyMeasureOrDensity, algorithm::OptimAlg, (result = result_mode, result_trafo = transformed_mode, trafo = trafo, #=trace_trafo = trace_trafo,=# info = r_optim) end -function _optim_minimize(f::Function, x_init::AbstractArray{<:Real}, algorithm::Optim.ZerothOrderOptimizer, ::BATContext) - opts = Optim.Options(store_trace = true, extended_trace=true) +function _optim_minimize(f::Function, x_init::AbstractArray{<:Real}, algorithm::Optim.ZerothOrderOptimizer, opts::Optim.Options, ::BATContext) _optim_optimize(f, x_init, algorithm, opts) end -function _optim_minimize(f::Function, x_init::AbstractArray{<:Real}, algorithm::Optim.FirstOrderOptimizer, context::BATContext) +function _optim_minimize(f::Function, x_init::AbstractArray{<:Real}, algorithm::Optim.FirstOrderOptimizer, opts::Optim.Options, context::BATContext) adsel = get_adselector(context) if adsel isa _NoADSelected throw(ErrorException("$(nameof(typeof(algorithm))) requires an ADSelector to be specified in the BAT context")) end fg! = NLSolversFG!(f, adsel) - opts = Optim.Options(store_trace = true, extended_trace=true) _optim_optimize(Optim.only_fg!(fg!), x_init, algorithm, opts) end diff --git a/src/extdefs/optim_defs.jl b/src/extdefs/optim_defs.jl index 656fb2957..092b8119d 100644 --- a/src/extdefs/optim_defs.jl +++ b/src/extdefs/optim_defs.jl @@ -29,10 +29,12 @@ $(TYPEDFIELDS) """ @with_kw struct OptimAlg{ ALG, + OPTS, TR<:AbstractTransformTarget, IA<:InitvalAlgorithm } <: AbstractModeEstimator optalg::ALG = ext_default(pkgext(Val(:Optim)), Val(:DEFAULT_OPTALG)) + options::OPTS = ext_default(pkgext(Val(:Optim)), Val(:DEFAULT_OPTS)) trafo::TR = PriorToGaussian() init::IA = InitFromTarget() end From fb70273d1d2b8bb0d2e224d4399cc4d6561476ce Mon Sep 17 00:00:00 2001 From: Cornelius-G Date: Fri, 6 Oct 2023 10:01:10 +0200 Subject: [PATCH 2/6] add wrapper for Optimization.jl, allow to pass kwargs --- Project.toml | 7 ++++ ext/BATOptimExt.jl | 22 ++++++++-- ext/BATOptimizationExt.jl | 72 ++++++++++++++++++++++++++++++++ src/BAT.jl | 1 + src/extdefs/extdefs.jl | 1 + src/extdefs/optim_defs.jl | 7 +++- src/extdefs/optimization_defs.jl | 44 +++++++++++++++++++ 7 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 ext/BATOptimizationExt.jl create mode 100644 src/extdefs/optimization_defs.jl diff --git a/Project.toml b/Project.toml index 22e75649d..254bca4bd 100644 --- a/Project.toml +++ b/Project.toml @@ -63,12 +63,14 @@ ValueShapes = "136a8f8c-c49b-4edb-8b98-f3d64d48be8f" ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" [weakdeps] +ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" AdvancedHMC = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d" Cuba = "8a292aeb-7a57-582c-b821-06e4c11590b1" Folds = "41a02a25-b8f0-4f67-bc48-60067656b558" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" NestedSamplers = "41ceaf6f-1696-4a54-9b49-2e7a9ec3782e" Optim = "429524aa-4258-5aef-a3af-852621145aeb" +Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Transducers = "28d57a85-8fef-5791-bfe6-a80928e7c999" UltraNest = "6822f173-b0be-4018-9ee2-28bf56348d09" @@ -80,10 +82,12 @@ BATFoldsExt = ["Folds", "Transducers"] BATHDF5Ext = "HDF5" BATNestedSamplersExt = "NestedSamplers" BATOptimExt = "Optim" +BATOptimizationExt = ["Optimization", "ADTypes"] BATPlotsExt = "Plots" BATUltraNestExt = "UltraNest" [compat] +ADTypes = "0.1, 0.2" Accessors = "0.1" AdvancedHMC = "0.5" AffineMaps = "0.2.3" @@ -124,6 +128,7 @@ Measurements = "2" NamedArrays = "0.9, 0.10" NestedSamplers = "0.8" Optim = "0.19,0.20, 0.21, 0.22, 1" +Optimization = "3" PDMats = "0.9, 0.10, 0.11" ParallelProcessingTools = "0.4" Parameters = "0.12" @@ -147,12 +152,14 @@ ZygoteRules = "0.2" julia = "1.6" [extras] +ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" AdvancedHMC = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d" Cuba = "8a292aeb-7a57-582c-b821-06e4c11590b1" Folds = "41a02a25-b8f0-4f67-bc48-60067656b558" HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" NestedSamplers = "41ceaf6f-1696-4a54-9b49-2e7a9ec3782e" Optim = "429524aa-4258-5aef-a3af-852621145aeb" +Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Transducers = "28d57a85-8fef-5791-bfe6-a80928e7c999" UltraNest = "6822f173-b0be-4018-9ee2-28bf56348d09" diff --git a/ext/BATOptimExt.jl b/ext/BATOptimExt.jl index 7b856dd11..585007b86 100644 --- a/ext/BATOptimExt.jl +++ b/ext/BATOptimExt.jl @@ -31,8 +31,6 @@ BAT.ext_default(::BAT.PackageExtension{:Optim}, ::Val{:DEFAULT_OPTALG}) = Optim. BAT.ext_default(::BAT.PackageExtension{:Optim}, ::Val{:NELDERMEAD_ALG}) = Optim.NelderMead() BAT.ext_default(::BAT.PackageExtension{:Optim}, ::Val{:LBFGS_ALG}) = Optim.LBFGS() -BAT.ext_default(::BAT.PackageExtension{:Optim}, ::Val{:DEFAULT_OPTS}) = Optim.Options(store_trace = true, extended_trace=true) - struct NLSolversFG!{F,AD} <: Function f::F ad::AD @@ -61,6 +59,21 @@ function (fg!::NLSolversFG!)(::Nothing, grad_f::AbstractVector{<:Real}, x::Abstr end +function convert_options(algorithm::OptimAlg) + if algorithm.abstol != NaN + @warn "The option 'abstol' is not used for this algorithm." + end + + kwargs = algorithm.kwargs + + algopts = (; iterations = algorithm.maxiters, time_limit = algorithm.maxtime, f_tol = algorithm.reltol,) + algopts = (; algopts..., kwargs...) + algopts = (; algopts..., store_trace = true, extended_trace=true) + + return Optim.Options(; algopts...) +end + + function BAT.bat_findmode_impl(target::AnyMeasureOrDensity, algorithm::OptimAlg, context::BATContext) transformed_density, trafo = transform_and_unshape(algorithm.trafo, target, context) inv_trafo = inverse(trafo) @@ -70,7 +83,10 @@ function BAT.bat_findmode_impl(target::AnyMeasureOrDensity, algorithm::OptimAlg, # Maximize density of original target, but run in transformed space, don't apply LADJ: f = fchain(inv_trafo, logdensityof(target), -) - optim_result = _optim_minimize(f, x_init, algorithm.optalg, algorithm.options, context) + + opts = convert_options(algorithm) + + optim_result = _optim_minimize(f, x_init, algorithm.optalg, opts, context) r_optim = Optim.MaximizationWrapper(optim_result) transformed_mode = Optim.minimizer(r_optim.res) result_mode = inv_trafo(transformed_mode) diff --git a/ext/BATOptimizationExt.jl b/ext/BATOptimizationExt.jl new file mode 100644 index 000000000..f7b3d14e1 --- /dev/null +++ b/ext/BATOptimizationExt.jl @@ -0,0 +1,72 @@ +# This file is a part of BAT.jl, licensed under the MIT License (MIT). + +module BATOptimizationExt + +@static if isdefined(Base, :get_extension) + import Optimization +else + import ..Optimization +end + +using BAT +BAT.pkgext(::Val{:Optimization}) = BAT.PackageExtension{:Optimization}() + + +using Random +using DensityInterface, ChangesOfVariables, InverseFunctions, FunctionChains +using HeterogeneousComputing, AutoDiffOperators +using StructArrays, ArraysOfArrays, ADTypes + +using BAT: AnyMeasureOrDensity, AbstractMeasureOrDensity + +using BAT: get_context, get_adselector, _NoADSelected +using BAT: bat_initval, transform_and_unshape, apply_trafo_to_init +using BAT: negative + + +AbstractModeEstimator(optalg::Any) = OptimizationAlg(optalg) +convert(::Type{AbstractModeEstimator}, alg::OptimizationAlg) = alg.optalg + +BAT.ext_default(::BAT.PackageExtension{:Optimization}, ::Val{:DEFAULT_OPTALG}) = nothing #Optim.NelderMead() + + +function build_optimizationfunction(f, adsel::AutoDiffOperators.ADSelector) + adm = convert_ad(ADTypes.AbstractADType, adsel) + optimization_function = Optimization.OptimizationFunction(f, adm) + return optimization_function +end + +function build_optimizationfunction(f, adsel::BAT._NoADSelected) + optimization_function = Optimization.OptimizationFunction(f) + return optimization_function +end + + +function BAT.bat_findmode_impl(target::AnyMeasureOrDensity, algorithm::OptimizationAlg, context::BATContext) + transformed_density, trafo = transform_and_unshape(algorithm.trafo, target, context) + inv_trafo = inverse(trafo) + + initalg = apply_trafo_to_init(trafo, algorithm.init) + x_init = collect(bat_initval(transformed_density, initalg, context).result) + + # Maximize density of original target, but run in transformed space, don't apply LADJ: + f = fchain(inv_trafo, logdensityof(target), -) + f2 = (x, p) -> f(x) + + adsel = get_adselector(context) + + optimization_function = build_optimizationfunction(f2, adsel) + optimization_problem = Optimization.OptimizationProblem(optimization_function, x_init) + + algopts = (maxiters = algorithm.maxiters, maxtime = algorithm.maxtime, abstol = algorithm.abstol, reltol = algorithm.reltol) + optimization_result = Optimization.solve(optimization_problem, algorithm.optalg; algopts..., algorithm.kwargs...) + + transformed_mode = optimization_result.u + result_mode = inv_trafo(transformed_mode) + + (result = result_mode, result_trafo = transformed_mode, trafo = trafo, info = optimization_result) +end + + + +end # module BATOptimizationExt diff --git a/src/BAT.jl b/src/BAT.jl index eb7142f5e..e7452443b 100644 --- a/src/BAT.jl +++ b/src/BAT.jl @@ -113,6 +113,7 @@ function __init__() @require HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" include("../ext/BATHDF5Ext.jl") @require NestedSamplers = "41ceaf6f-1696-4a54-9b49-2e7a9ec3782e" include("../ext/BATNestedSamplersExt.jl") @require Optim = "429524aa-4258-5aef-a3af-852621145aeb" include("../ext/BATOptimExt.jl") + @require Optimization = "429524aa-4258-5aef-a3af-852621145aeb" @require ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" include("../ext/BATOptimizationExt.jl") @require Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" include("../ext/BATPlotsExt.jl") @require UltraNest = "6822f173-b0be-4018-9ee2-28bf56348d09" include("../ext/BATUltraNestExt.jl") end diff --git a/src/extdefs/extdefs.jl b/src/extdefs/extdefs.jl index 19f7ba2f2..516a1295e 100644 --- a/src/extdefs/extdefs.jl +++ b/src/extdefs/extdefs.jl @@ -4,4 +4,5 @@ include("advancedhmc_defs.jl") include("cuba_defs.jl") include("nestedsamplers_defs.jl") include("optim_defs.jl") +include("optimization_defs.jl") include("ultranest_defs.jl") diff --git a/src/extdefs/optim_defs.jl b/src/extdefs/optim_defs.jl index 092b8119d..c993a2a2a 100644 --- a/src/extdefs/optim_defs.jl +++ b/src/extdefs/optim_defs.jl @@ -29,13 +29,16 @@ $(TYPEDFIELDS) """ @with_kw struct OptimAlg{ ALG, - OPTS, TR<:AbstractTransformTarget, IA<:InitvalAlgorithm } <: AbstractModeEstimator optalg::ALG = ext_default(pkgext(Val(:Optim)), Val(:DEFAULT_OPTALG)) - options::OPTS = ext_default(pkgext(Val(:Optim)), Val(:DEFAULT_OPTS)) trafo::TR = PriorToGaussian() init::IA = InitFromTarget() + maxiters::Int = 1_000 + maxtime::Float64 = NaN + abstol::Float64 = NaN + reltol::Float64 = 0.0 + kwargs::NamedTuple = (;) end export OptimAlg diff --git a/src/extdefs/optimization_defs.jl b/src/extdefs/optimization_defs.jl new file mode 100644 index 000000000..f87f17c25 --- /dev/null +++ b/src/extdefs/optimization_defs.jl @@ -0,0 +1,44 @@ +# This file is a part of BAT.jl, licensed under the MIT License (MIT). + + +""" + OptimizationAlg + +Selects an optimization algorithm from the +[Optimization.jl](https://github.com/SciML/Optimization.jl) +package. + +Note that when using first order algorithms like `OptimizationOptimJL.LBFGS`, your +[`BATContext`](@ref) needs to include an `ADSelector` that specifies +which automatic differentiation backend should be used. + +Constructors: + +* ```$(FUNCTIONNAME)(; fields...)``` + +`optalg` must be an `Optimization.AbstractOptimizer`. + +Fields: + +$(TYPEDFIELDS) + +!!! note + + This algorithm is only available if the Optimization package is loaded (e.g. via + `import Optimization`. +""" +@with_kw struct OptimizationAlg{ + ALG, + TR<:AbstractTransformTarget, + IA<:InitvalAlgorithm +} <: AbstractModeEstimator + optalg::ALG = ext_default(pkgext(Val(:Optimization)), Val(:DEFAULT_OPTALG)) + trafo::TR = PriorToGaussian() + init::IA = InitFromTarget() + maxiters::Int64 = 1_000 + maxtime::Float64 = NaN + abstol::Float64 = NaN + reltol::Float64 = 0.0 + kwargs::NamedTuple = (;) +end +export OptimizationAlg From eab3567fb84b5738a2af3d93c8db52a74839c8b7 Mon Sep 17 00:00:00 2001 From: Cornelius-G Date: Thu, 12 Oct 2023 13:59:03 +0200 Subject: [PATCH 3/6] add some tests for new optimization interface --- .gitignore | 1 + examples/dev-internal/test_findmode.jl | 116 ++++++++++++++++++++++ ext/BATOptimExt.jl | 4 +- src/extdefs/optimization_defs.jl | 4 +- test/Project.toml | 1 + test/optimization/test_mode_estimators.jl | 43 +++++++- 6 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 examples/dev-internal/test_findmode.jl diff --git a/.gitignore b/.gitignore index 318e444db..719614c32 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.jl.mem .ipynb_checkpoints Manifest.toml +.vscode/settings.json diff --git a/examples/dev-internal/test_findmode.jl b/examples/dev-internal/test_findmode.jl new file mode 100644 index 000000000..43b7ade2b --- /dev/null +++ b/examples/dev-internal/test_findmode.jl @@ -0,0 +1,116 @@ +using BAT +using Optim + +posterior = BAT.example_posterior() + +optalg = OptimAlg(; + optalg = Optim.NelderMead(parameters=Optim.FixedParameters()), + maxiters=200, + kwargs = (f_calls_limit=100,), +) + +my_mode = bat_findmode(posterior, optalg) + +fieldnames(typeof(my_mode.info.res)) + + +using BAT +#using Optim +#using Optimization +using OptimizationOptimJL + +using InverseFunctions, FunctionChains, DensityInterface + + +posterior = BAT.example_posterior() +optalg = OptimizationAlg(; optalg = OptimizationOptimJL.ParticleSwarm(n_particles=10), maxiters=200, kwargs=(f_calls_limit=500,)) +my_result = bat_findmode(posterior, optalg) + +a = my_result.info + +@test a.cache.solver_args.maxiters == 500 + +dump(a.alg) + +fieldnames(typeof(a.cache.solver_args)) + +fieldnames(typeof(a.original.method)) + +my_mode.info.original + + + + +# Define a NamedTuple with keyword arguments +nt = (a=1, b=2) + +# Define a function that accepts keyword arguments +function my_function(; a=0, b=0, c=0) + println("a = $a") + println("b = $b") + println("c = $c") +end + +# Call the function and unpack the NamedTuple +my_function(; nt...) + + + + + + + + + + + + + + + + + + + +context = get_batcontext() +target = posterior +transformed_density, trafo = BAT.transform_and_unshape(PriorToGaussian(), target, context) +inv_trafo = inverse(trafo) +initalg = BAT.apply_trafo_to_init(trafo, InitFromTarget()) +x_init = collect(bat_initval(transformed_density, initalg, context).result) + +f = fchain(inv_trafo, logdensityof(target), -) +f2 = (x, p) -> f(x) + + +optimization_function = Optimization.OptimizationFunction(f2, Optimization.SciMLBase.NoAD()) +optimization_problem = Optimization.OptimizationProblem(optimization_function, x_init) +optimization_result = Optimization.solve(optimization_problem,OptimizationOptimJL.NelderMead()) + + +optalg = OptimizationAlg(;optalg = OptimizationOptimJL.NelderMead()) +my_mode = bat_findmode(posterior, optalg) + +my_mode.info.original +fieldnames(typeof(my_mode.info)) + +rosenbrock(x, p) = (p[1] - x[1])^2 + p[2] * (x[2] - x[1]^2)^2 +f = rosenbrock + + +using AutoDiffOperators + +b = Optimization.SciMLBase.NoAD() +supertype(typeof(b)) + +adm = ADModule(:ForwardDiff) + +adsel = BAT.get_adselector(context) +supertype(typeof(adsel)) + + + +adm2 = convert_ad(ADTypes.AbstractADType, adm) +ADTypes.AutoForwardDiff() + +optimization_function = Optimization.OptimizationFunction(f2, adm2) \ No newline at end of file diff --git a/ext/BATOptimExt.jl b/ext/BATOptimExt.jl index 585007b86..9eb0f17f3 100644 --- a/ext/BATOptimExt.jl +++ b/ext/BATOptimExt.jl @@ -60,8 +60,8 @@ end function convert_options(algorithm::OptimAlg) - if algorithm.abstol != NaN - @warn "The option 'abstol' is not used for this algorithm." + if !isnan(algorithm.abstol) + @warn "The option 'abstol=$(algorithm.abstol)' is not used for this algorithm." end kwargs = algorithm.kwargs diff --git a/src/extdefs/optimization_defs.jl b/src/extdefs/optimization_defs.jl index f87f17c25..9805580f2 100644 --- a/src/extdefs/optimization_defs.jl +++ b/src/extdefs/optimization_defs.jl @@ -24,8 +24,8 @@ $(TYPEDFIELDS) !!! note - This algorithm is only available if the Optimization package is loaded (e.g. via - `import Optimization`. + This algorithm is only available if the `Optimization` package or any of its submodules, like `OptimizationOptimJL`, is loaded (e.g. via + `import Optimization`). """ @with_kw struct OptimizationAlg{ ALG, diff --git a/test/Project.toml b/test/Project.toml index da68533cd..d3183dbd3 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -27,6 +27,7 @@ Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" NestedSamplers = "41ceaf6f-1696-4a54-9b49-2e7a9ec3782e" Optim = "429524aa-4258-5aef-a3af-852621145aeb" +OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e" PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" ParallelProcessingTools = "8e8a01fc-6193-5ca1-a2f1-20776dae4199" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" diff --git a/test/optimization/test_mode_estimators.jl b/test/optimization/test_mode_estimators.jl index af9c1bba6..303a9b67a 100644 --- a/test/optimization/test_mode_estimators.jl +++ b/test/optimization/test_mode_estimators.jl @@ -4,7 +4,7 @@ using Test using LinearAlgebra, Distributions, StatsBase, ValueShapes, Random123, DensityInterface using UnPack, InverseFunctions using AutoDiffOperators, ForwardDiff -using Optim +using Optim, OptimizationOptimJL @testset "mode_estimators" begin prior = NamedTupleDist( @@ -71,17 +71,54 @@ using Optim end - @testset "NelderMead" begin + @testset "Optim.jl - NelderMead" begin context = BATContext(rng = Philox4x((0, 0))) test_findmode(posterior, OptimAlg(optalg = NelderMead(), trafo = DoNotTransform()), 0.01, context) end + @testset "Optim.jl with custom options" begin # checks that options are correctly passed to Optim.jl + context = BATContext(rng = Philox4x((0, 0))) + optimizer = OptimAlg(optalg = NelderMead(), trafo = DoNotTransform(), maxiters=20, maxtime=30, reltol=0.2, kwargs=(f_calls_limit=500,)) + + result = bat_findmode(posterior, optimizer, context) + @test result.info.res.iterations == 20 + @test result.info.res.time_limit == 30 + @test result.info.res.f_reltol == 0.2 + @test result.info.res.f_calls == 500 + + end - @testset "LBFGS" begin + @testset "Optim.jl - LBFGS" begin context = BATContext(rng = Philox4x((0, 0)), ad = ADModule(:ForwardDiff)) # Result Optim.maximize with LBFGS is not type-stable: test_findmode(posterior, OptimAlg(optalg = LBFGS(), trafo = DoNotTransform()), 0.01, inferred = false, context) test_findmode_ctx(posterior, OptimAlg(optalg = LBFGS(), trafo = DoNotTransform()), 0.01, context) end + + + @testset "Optimization.jl - NelderMead" begin + context = BATContext(rng = Philox4x((0, 0))) + # result is not type-stable: + test_findmode(posterior, OptimizationAlg(optalg = OptimizationOptimJL.NelderMead(), trafo = DoNotTransform()), 0.01, context, inferred = false) + end + + @testset "Optimization.jl with custom options" begin # checks that options are correctly passed to Optimization.jl + context = BATContext(rng = Philox4x((0, 0))) + optimizer = OptimizationAlg(optalg = OptimizationOptimJL.ParticleSwarm(n_particles=10), maxiters=200, kwargs=(f_calls_limit=500,), trafo=DoNotTransform()) + + # result is not type-stable: + test_findmode(posterior, optimizer, 0.01, context, inferred = false) + + optimizer = OptimizationAlg(optalg = OptimizationOptimJL.ParticleSwarm(n_particles=10), + maxiters=200, maxtime=30, reltol=0.2, kwargs=(f_calls_limit=500,), trafo=DoNotTransform()) + + result = bat_findmode(posterior, optimizer, context) + @test result.info.cache.solver_args.maxiters == 200 + @test result.info.cache.solver_args.f_calls_limit == 500 + @test result.info.cache.solver_args.reltol == 0.2 + @test result.info.cache.solver_args.maxtime == 30 + @test result.info.original.method.n_particles == 10 + end + end From 6cca5a917b8ce78c4f4fa873f2004a9bb18d4c71 Mon Sep 17 00:00:00 2001 From: Cornelius-G Date: Wed, 28 Feb 2024 14:19:05 +0100 Subject: [PATCH 4/6] fix tests and documentation --- ext/BATOptimizationExt.jl | 4 ++-- src/extdefs/optimization_defs.jl | 2 ++ src/samplers/importance/importance_sampler.jl | 2 +- test/optimization/test_mode_estimators.jl | 6 +++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ext/BATOptimizationExt.jl b/ext/BATOptimizationExt.jl index f7b3d14e1..3405a87ca 100644 --- a/ext/BATOptimizationExt.jl +++ b/ext/BATOptimizationExt.jl @@ -51,11 +51,11 @@ function BAT.bat_findmode_impl(target::AnyMeasureOrDensity, algorithm::Optimizat # Maximize density of original target, but run in transformed space, don't apply LADJ: f = fchain(inv_trafo, logdensityof(target), -) - f2 = (x, p) -> f(x) + target_f = (x, p) -> f(x) adsel = get_adselector(context) - optimization_function = build_optimizationfunction(f2, adsel) + optimization_function = build_optimizationfunction(target_f, adsel) optimization_problem = Optimization.OptimizationProblem(optimization_function, x_init) algopts = (maxiters = algorithm.maxiters, maxtime = algorithm.maxtime, abstol = algorithm.abstol, reltol = algorithm.reltol) diff --git a/src/extdefs/optimization_defs.jl b/src/extdefs/optimization_defs.jl index 9805580f2..1429bcecf 100644 --- a/src/extdefs/optimization_defs.jl +++ b/src/extdefs/optimization_defs.jl @@ -17,6 +17,8 @@ Constructors: * ```$(FUNCTIONNAME)(; fields...)``` `optalg` must be an `Optimization.AbstractOptimizer`. +The field `kwargs` can be used to pass additional keywords to the optimizers +See the [Optimization.jl documentation](https://docs.sciml.ai/Optimization/stable/) for the available keyword arguments. Fields: diff --git a/src/samplers/importance/importance_sampler.jl b/src/samplers/importance/importance_sampler.jl index 4c175d66b..de0e40132 100644 --- a/src/samplers/importance/importance_sampler.jl +++ b/src/samplers/importance/importance_sampler.jl @@ -60,7 +60,7 @@ function bat_sample_impl( vol = exp(BigFloat(log_volume(spatialvolume(var_bounds(density))))) est_integral = mean(weights) * vol # ToDo: Add integral error estimate - + @show samples samples_trafo = shape.(DensitySampleVector(samples, logvals, weight = weights)) samples_notrafo = inverse(trafo).(samples_trafo) diff --git a/test/optimization/test_mode_estimators.jl b/test/optimization/test_mode_estimators.jl index 303a9b67a..ece5af389 100644 --- a/test/optimization/test_mode_estimators.jl +++ b/test/optimization/test_mode_estimators.jl @@ -78,13 +78,13 @@ using Optim, OptimizationOptimJL @testset "Optim.jl with custom options" begin # checks that options are correctly passed to Optim.jl context = BATContext(rng = Philox4x((0, 0))) - optimizer = OptimAlg(optalg = NelderMead(), trafo = DoNotTransform(), maxiters=20, maxtime=30, reltol=0.2, kwargs=(f_calls_limit=500,)) + optimizer = OptimAlg(optalg = NelderMead(), trafo = DoNotTransform(), maxiters=20, maxtime=30, reltol=0.2, kwargs=(f_calls_limit=25,)) result = bat_findmode(posterior, optimizer, context) - @test result.info.res.iterations == 20 + @test result.info.res.iterations <= 20 @test result.info.res.time_limit == 30 @test result.info.res.f_reltol == 0.2 - @test result.info.res.f_calls == 500 + @test result.info.res.f_calls <= 26 end From d8dff9fc4cbb4cb24a998f8c338ddf57d844d7a2 Mon Sep 17 00:00:00 2001 From: Cornelius-G Date: Thu, 29 Feb 2024 11:11:56 +0100 Subject: [PATCH 5/6] add OptimizationAlg to docs --- docs/src/list_of_algorithms.md | 16 ++++++++++++++++ docs/src/stable_api.md | 1 + 2 files changed, 17 insertions(+) diff --git a/docs/src/list_of_algorithms.md b/docs/src/list_of_algorithms.md index 4688a3800..4716fec16 100644 --- a/docs/src/list_of_algorithms.md +++ b/docs/src/list_of_algorithms.md @@ -167,6 +167,22 @@ bat_findmode(target, OptimAlg(optalg = Optim.LBFGS())) Requires the [Optim](https://github.com/JuliaNLSolvers/Optim.jl) Julia package to be loaded explicitly. +### Optimization.jl Optimization Algorithms + +BAT mode finding algorithm type: [`OptimizationAlg`](@ref). + +```julia +using OptimizationOptimJL + +alg = OptimizationAlg(; + optalg = OptimizationOptimJL.ParticleSwarm(n_particles=10), + maxiters=200, + kwargs=(f_calls_limit=50,) +) +bat_findmode(target, alg) +``` +Requires one of the [Optimization.jl](https://github.com/SciML/Optimization.jl) packages to be loaded explicitly. + ### Maximum Sample Estimator BAT mode finding algorithm type: [`MaxDensitySearch`](@ref) diff --git a/docs/src/stable_api.md b/docs/src/stable_api.md index e294572de..353c2d6de 100644 --- a/docs/src/stable_api.md +++ b/docs/src/stable_api.md @@ -92,6 +92,7 @@ MetropolisHastings MHProposalDistTuning ModeAsDefined OptimAlg +OptimizationAlg OrderedResampling PosteriorMeasure PriorSubstitution From 86b07b4dc56c573a20e0fef918e1aae2c3703988 Mon Sep 17 00:00:00 2001 From: Oliver Schulz Date: Tue, 19 Mar 2024 14:22:31 +0100 Subject: [PATCH 6/6] Fix codecov action --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2dbf6e31c..1023363d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,8 @@ jobs: - uses: codecov/codecov-action@v4 if: matrix.version == '1' && matrix.os == 'ubuntu-latest' && matrix.arch == 'x64' with: + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} file: lcov.info docs: name: Documentation