From e8b8f7b483cbef57b02928535641a2a335b231dc Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 11:02:45 +0100 Subject: [PATCH 01/20] Add AGP model --- botorch/models/gp_regression_multisource.py | 395 ++++++++++++++++++++ 1 file changed, 395 insertions(+) create mode 100644 botorch/models/gp_regression_multisource.py diff --git a/botorch/models/gp_regression_multisource.py b/botorch/models/gp_regression_multisource.py new file mode 100644 index 0000000000..c8f5797b22 --- /dev/null +++ b/botorch/models/gp_regression_multisource.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +r""" +Multi-Source Gaussian Process Regression models based on GPyTorch models. + +For more on Multi-Source BO, see the +`tutorial `__ + +BRIEF DESCRIPTION + +.. [Ca2021ms] + Candelieri, A., & Archetti, F. (2021). + Sparsifying to optimize over multiple information sources: + an augmented Gaussian process based algorithm. + Structural and Multidisciplinary Optimization, 64, 239-255. +""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from botorch import fit_gpytorch_mll +from botorch.exceptions import InputDataError +from botorch.models import FixedNoiseGP, SingleTaskGP +from botorch.models.transforms.input import InputTransform +from botorch.models.transforms.outcome import OutcomeTransform +from botorch.utils import draw_sobol_samples +from gpytorch import ExactMarginalLogLikelihood, Module +from gpytorch.likelihoods.likelihood import Likelihood +from gpytorch.means.mean import Mean +from gpytorch.models import ExactGP +from torch import Tensor + + +def get_random_x_for_agp( + n: int, + bounds: Tensor, + q: int, + seed: Optional[int] = None, +): + r"""Draw qMC samples from the box defined by bounds. + The function assures that at least one point belong to + the source 0 (the ground truth). + + Args: + n: The number of samples. + bounds: A `2 x d` dimensional tensor specifying box constraints on a + `d`-dimensional space, where bounds[0, :] and bounds[1, :] correspond + to lower and upper bounds, respectively. The last dimension represents + number of sources. + q: The size of each q-batch. + seed: The seed used for initializing Owen scrambling. If None (default), + use a random seed. + + Returns: + A `n x q x d`-dim tensor of qMC samples from the box + defined by bounds. + """ + if n < 1: + raise InputDataError( + f"The number of points n should be greater than 0 (given n={n})." + ) + train_x = draw_sobol_samples(bounds=bounds, n=n, q=q, seed=seed).squeeze(1) + train_x[..., -1] = torch.round(train_x[..., -1], decimals=0) + if 0 not in train_x[..., -1]: + true_idxs = torch.randint(0, n, [max(1, int(n * 0.2))]) + train_x[true_idxs, -1] = 0 + return train_x + + +class SingleTaskAugmentedGP(SingleTaskGP): + r"""A single-task multi-source GP model. + + The Augmented Gaussian Process is described in [Ca2021ms]_. + """ + + def __init__( + self, + train_X: Tensor, + train_Y: Tensor, + train_Yvar: Optional[Tensor] = None, + m: int = 1, + likelihood: Optional[Likelihood] = None, + covar_module: Optional[Module] = None, + mean_module: Optional[Mean] = None, + outcome_transform: Optional[OutcomeTransform] = None, + input_transform: Optional[InputTransform] = None, + ) -> None: + r""" + Args: + train_X: A `batch_shape x n x (d + 1)` tensor of training features, + where the additional dimension is for the source parameter. + train_Y: A `batch_shape x n x m` tensor of training observations. + train_Yvar: A `batch_shape x n x m` tensor of observed measurement + noise. + m: The moltiplicator factor of the model standard deviation used to select + points from other sources to add to the Augmented GP. + likelihood: A likelihood. If omitted, use a standard + GaussianLikelihood with inferred noise level. + covar_module: The module computing the covariance (Kernel) matrix. + If omitted, use a `MaternKernel`. + mean_module: The mean function to be used. If omitted, use a + `ConstantMean`. + outcome_transform: An outcome transform that is applied to the + training data during instantiation and to the posterior during + inference (that is, the `Posterior` obtained by calling + `.posterior` on the model will be on the original scale). + input_transform: An input transform that is applied in the model's + forward pass. + """ + if m <= 0: + raise InputDataError(f"The value of m must be greater than 0, given m={m}.") + if 0 not in train_X[..., -1]: + raise InputDataError( + "At least one observation of the true source have to be provided." + ) + # Divide train_X and train_Y based on the source + train_S = train_X[..., -1] + sources = torch.unique(train_S).int() + if sources.shape[0] == 1: + raise InputDataError("AGP is meant to be used with more than one source.") + train_X = [train_X[torch.where(train_S == s)] for s in sources] + train_Y = [train_Y[torch.where(train_S == s)] for s in sources] + self.n_true_points = len(train_X[-1]) + self.max_n_cheap_points = max([len(points) for points in train_X[:-1]]) + + # Init and fit a SingleTaskGP for each source + self.models = [ + self._init_fit_gp( + x[:, :-1], + y, + likelihood, + covar_module, + mean_module, + outcome_transform, + input_transform, + ) + for x, y in zip(train_X, train_Y) + ] + + # Create the training set for the AGP selecting all + # the observations from the high fidelity source + # and the reliable observations from the other sources + reliable_idxs = [ + _get_reliable_observations( + self.models[0], self.models[s], train_X[s][:, :-1], m + ) + for s in sources[1:] + ] + train_X = torch.cat( + [ + train_X[s] if s == 0 else train_X[s][reliable_idxs[s - 1]] + for s in sources + ] + )[:, :-1] + train_Y = torch.cat( + [ + train_Y[s] if s == 0 else train_Y[s][reliable_idxs[s - 1]] + for s in sources + ] + ) + + super().__init__( + train_X, + train_Y, + train_Yvar, + likelihood, + covar_module, + mean_module, + outcome_transform, + input_transform, + ) + + def _init_fit_gp( + self, + train_X: Tensor, + train_Y: Tensor, + likelihood: Optional[Likelihood] = None, + covar_module: Optional[Module] = None, + mean_module: Optional[Mean] = None, + outcome_transform: Optional[OutcomeTransform] = None, + input_transform: Optional[InputTransform] = None, + ) -> SingleTaskGP: + r"""Initialize and fit a Single Task GP model. + + Args: + train_X: A `batch_shape x n x d` tensor of training features. + train_Y: A `batch_shape x n x m` tensor of training observations. + likelihood: A likelihood. If omitted, use a standard + GaussianLikelihood with inferred noise level. + covar_module: The module computing the covariance (Kernel) matrix. + If omitted, use a `MaternKernel`. + mean_module: The mean function to be used. If omitted, use a + `ConstantMean`. + outcome_transform: An outcome transform that is applied to the + training data during instantiation and to the posterior during + inference (that is, the `Posterior` obtained by calling + `.posterior` on the model will be on the original scale). + input_transform: An input transform that is applied in the model's + forward pass. + + Returns: + The fitted Single Task GP and its Marginal Log Likelihood. + """ + gp = SingleTaskGP( + train_X, + train_Y, + likelihood=likelihood, + covar_module=covar_module, + mean_module=mean_module, + outcome_transform=outcome_transform, + input_transform=input_transform, + ) + mll = ExactMarginalLogLikelihood(gp.likelihood, gp) + fit_gpytorch_mll(mll) + return gp + + +class FixedNoiseAugmentedGP(FixedNoiseGP): + def __init__( + self, + train_X: Tensor, + train_Y: Tensor, + train_Yvar: Tensor, + m: int = 1, + covar_module: Optional[Module] = None, + mean_module: Optional[Mean] = None, + outcome_transform: Optional[OutcomeTransform] = None, + input_transform: Optional[InputTransform] = None, + ) -> None: + """ + Args: + train_X: A `batch_shape x n x (d + 1)` tensor of training features, + where the additional dimension is for the source parameter. + train_Y: A `batch_shape x n x m` tensor of training observations. + train_Yvar: A `batch_shape x n x m` tensor of observed measurement + noise. + m: The moltiplicator factor of the model standard deviation used to select + points from other sources to add to the Augmented GP. + covar_module: The module computing the covariance (Kernel) matrix. + If omitted, use a `MaternKernel`. + mean_module: The mean function to be used. If omitted, use a + `ConstantMean`. + outcome_transform: An outcome transform that is applied to the + training data during instantiation and to the posterior during + inference (that is, the `Posterior` obtained by calling + `.posterior` on the model will be on the original scale). + input_transform: An input transform that is applied in the model's + forward pass. + """ + if m <= 0: + raise InputDataError(f"The value of m must be greater than 0, given m={m}.") + if 0 not in train_X[..., -1]: + raise InputDataError( + "At least one observation of the true source have to be provided." + ) + # Divide train_X and train_Y based on the source + train_S = train_X[..., -1] + sources = torch.unique(train_S).int() + if sources.shape[0] == 1: + raise InputDataError("AGP is meant to be used with more than one source.") + + train_X = [train_X[torch.where(train_S == s)] for s in sources] + train_Y = [train_Y[torch.where(train_S == s)] for s in sources] + train_Yvar = [train_Yvar[torch.where(train_S == s)] for s in sources] + self.n_true_points = len(train_X[-1]) + self.max_n_cheap_points = max([len(points) for points in train_X[:-1]]) + + # Init and fit a SingleTaskGP for each source + self.models = [ + self._init_fit_gp( + x[:, :-1], + y, + yvar, + covar_module, + mean_module, + outcome_transform, + input_transform, + ) + for x, y, yvar in zip(train_X, train_Y, train_Yvar) + ] + + # Create the training set for the AGP selecting all the + # observations from the high fidelity source + # and the reliable observations from the other sources + reliable_idxs = [ + _get_reliable_observations( + self.models[0], self.models[s], train_X[s][:, :-1], m + ) + for s in sources[1:] + ] + train_X = torch.cat( + [ + train_X[s] if s == 0 else train_X[s][reliable_idxs[s - 1]] + for s in sources + ] + )[:, :-1] + train_Y = torch.cat( + [ + train_Y[s] if s == 0 else train_Y[s][reliable_idxs[s - 1]] + for s in sources + ] + ) + train_Yvar = torch.cat( + [ + train_Yvar[s] if s == 0 else train_Yvar[s][reliable_idxs[s - 1]] + for s in sources + ] + ) + + super().__init__( + train_X, + train_Y, + train_Yvar, + covar_module, + mean_module, + outcome_transform, + input_transform, + ) + + def _init_fit_gp( + self, + train_X: Tensor, + train_Y: Tensor, + train_Yvar: Tensor, + covar_module: Optional[Module] = None, + mean_module: Optional[Mean] = None, + outcome_transform: Optional[OutcomeTransform] = None, + input_transform: Optional[InputTransform] = None, + ) -> FixedNoiseGP: + r"""Initialize and fit a Fixed Noise GP model. + + Args: + train_X: A `batch_shape x n x d` tensor of training features. + train_Y: A `batch_shape x n x m` tensor of training observations. + train_Yvar: A `batch_shape x n x m` tensor of observed measurement + noise. + covar_module: The module computing the covariance (Kernel) matrix. + If omitted, use a `MaternKernel`. + mean_module: The mean function to be used. If omitted, use a + `ConstantMean`. + outcome_transform: An outcome transform that is applied to the + training data during instantiation and to the posterior during + inference (that is, the `Posterior` obtained by calling + `.posterior` on the model will be on the original scale). + input_transform: An input transform that is applied in the model's + forward pass. + Returns: + The fitted Fixed Noise GP and its Marginal Log Likelihood. + """ + gp = FixedNoiseGP( + train_X, + train_Y, + train_Yvar=train_Yvar, + covar_module=covar_module, + mean_module=mean_module, + outcome_transform=outcome_transform, + input_transform=input_transform, + ) + mll = ExactMarginalLogLikelihood(gp.likelihood, gp) + fit_gpytorch_mll(mll) + return gp + + +def _get_reliable_observations( + trusty_model: ExactGP, + other_model: ExactGP, + x: Tensor, + m: int = 1, +) -> Tensor: + r"""Get the points whose posterior mean computed with other_model + is inside m * trusty_model standard deviation. + + Args: + trusty_model: The GP model of the trust source. + other_model: The GP model of a lower fidelity source. + x: A `batch_shape x n x d` tensor of training features. + m: The moltiplicator factor of the model standard deviation used to select + points from other sources to add to the Augmented GP. + Returns: + A `batch_shape x N x d` tensor of reliable points + """ + m0_posterior = trusty_model.posterior(x) + m0_mu = torch.flatten(m0_posterior.mean) + m0_sigma = torch.sqrt(torch.flatten(m0_posterior.variance)) + m1_posterior = other_model.posterior(x) + m1_mu = torch.flatten(m1_posterior.mean) + + return torch.where(torch.abs(m0_mu - m1_mu) < m * m0_sigma)[0] From 32caa447fdec2327e2c279265c8f50976ab893b0 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 11:11:21 +0100 Subject: [PATCH 02/20] Add Augmented UCB acquisition function for multi-source --- botorch/acquisition/augmented_multi_source.py | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 botorch/acquisition/augmented_multi_source.py diff --git a/botorch/acquisition/augmented_multi_source.py b/botorch/acquisition/augmented_multi_source.py new file mode 100644 index 0000000000..827c02f90d --- /dev/null +++ b/botorch/acquisition/augmented_multi_source.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import annotations + +from typing import Dict, Optional, Tuple, Union + +import torch + +from botorch.acquisition import UpperConfidenceBound +from botorch.acquisition.objective import PosteriorTransform +from botorch.exceptions import UnsupportedError +from botorch.models.model import Model +from botorch.utils.transforms import t_batch_mode_transform +from gpytorch.models import ExactGP +from torch import Tensor + + +class AugmentedUpperConfidenceBound(UpperConfidenceBound): + r"""Single-outcome Multi-Source Upper Confidence Bound (UCB). + + Description... + + `AUCB(x, s, y^+) = + ((mu(x) + sqrt(beta) * sigma(x)) - y^+) / + (c(s) (1 + abs(mu(x) - mu_s(x))))`, + where `mu` and `sigma` are the posterior mean and standard deviation of the AGP, + `mu_s` is the posterior mean of the GP modelling the s-th source and + c(s) is the cost of the source s. + + Example: + >>> model = AugmentedSingleTaskGP(train_X, train_Y) + >>> UCB = AugmentedUpperConfidenceBound(model, beta=0.2) + >>> ucb = UCB(test_X) + """ + + def __init__( + self, + model: Model, + cost: Dict, + best_f: Union[float, Tensor], + beta: Union[float, Tensor], + posterior_transform: Optional[PosteriorTransform] = None, + maximize: bool = True, + ) -> None: + r"""Single-outcome Augmented Upper Confidence Bound. + + Args: + model: A fitted single-outcome Augmented GP model. + beta: Either a scalar or a one-dim tensor with `b` elements (batch mode) + representing the trade-off parameter between mean and covariance + posterior_transform: A PosteriorTransform. If using a multi-output model, + a PosteriorTransform that transforms the multi-output posterior into a + single-output posterior is required. + maximize: If True, consider the problem a maximization problem. + """ + if not hasattr(model, "models"): + raise UnsupportedError("Model have to be multi-source.") + super().__init__( + model=model, + beta=beta, + maximize=maximize, + posterior_transform=posterior_transform, + ) + self.cost = cost + self.best_f = best_f + + @t_batch_mode_transform(expected_q=1) + def forward(self, X: Tensor) -> Tensor: + r"""Evaluate the Upper Confidence Bound on the candidate set X. + + Args: + X: A `(b1 x ... bk) x 1 x d`-dim batched tensor of `d`-dim design points. + + Returns: + A `(b1 x ... bk)`-dim tensor of Augmented Upper Confidence Bound values at + the given design points `X`. + """ + alpha = torch.zeros(X.shape[0], dtype=X.dtype) + agp_mean, agp_sigma = self._mean_and_sigma(X[..., :-1]) + cb = (self.best_f if self.maximize else -self.best_f) + ( + (agp_mean if self.maximize else -agp_mean) + self.beta.sqrt() * agp_sigma + ) + source_idxs = { + int(s.tolist()): torch.where(torch.round(X[..., -1], decimals=0) == s)[0] + for s in torch.round(X[..., -1], decimals=0).unique() + } + for s in source_idxs: + mean, sigma = self._mean_and_sigma( + X[source_idxs[s], :, :-1], self.model.models[s] + ) + alpha[source_idxs[s]] = ( + cb[source_idxs[s]] + / self.cost[s] + * (1 + torch.abs(agp_mean[source_idxs[s]] - mean)) + ) + return alpha + + def _mean_and_sigma( + self, + X: Tensor, + model: ExactGP = None, + compute_sigma: bool = True, + min_var: float = 1e-12, + ) -> Tuple[Tensor, Optional[Tensor]]: + r"""Computes the first and second moments of the model posterior. + + Args: + X: `batch_shape x q x d`-dim Tensor of model inputs. + model: the model to use. If None, self is used. + compute_sigma: Boolean indicating whether to compute the second + moment (default: True). + min_var: The minimum value the variance is clamped too. Should be positive. + + Returns: + A tuple of tensors containing the first and second moments of the model + posterior. Removes the last two dimensions if they have size one. Only + returns a single tensor of means if compute_sigma is True. + """ + self.to(device=X.device) + if model is None: + posterior = self.model.posterior( + X=X, posterior_transform=self.posterior_transform + ) + else: + posterior = model.posterior( + X=X, posterior_transform=self.posterior_transform + ) + mean = posterior.mean.squeeze(-2).squeeze(-1) # removing redundant dimensions + if not compute_sigma: + return mean, None + sigma = posterior.variance.clamp_min(min_var).sqrt().view(mean.shape) + return mean, sigma From 6152e189c5169f137c2a847cebff2f8ac69c32b2 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 11:17:10 +0100 Subject: [PATCH 03/20] Add AGP unit test --- test/models/test_gp_regression_multisource.py | 475 ++++++++++++++++++ 1 file changed, 475 insertions(+) create mode 100644 test/models/test_gp_regression_multisource.py diff --git a/test/models/test_gp_regression_multisource.py b/test/models/test_gp_regression_multisource.py new file mode 100644 index 0000000000..8bf57627d1 --- /dev/null +++ b/test/models/test_gp_regression_multisource.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import itertools +import math +import warnings + +import torch + +from botorch import fit_gpytorch_mll +from botorch.exceptions import InputDataError, OptimizationWarning +from botorch.models import FixedNoiseGP, SingleTaskGP +from botorch.models.gp_regression_multisource import ( + _get_reliable_observations, + FixedNoiseAugmentedGP, + get_random_x_for_agp, + SingleTaskAugmentedGP, +) +from botorch.models.transforms import Normalize, Standardize +from botorch.posteriors import GPyTorchPosterior +from botorch.sampling import SobolQMCNormalSampler +from botorch.utils import draw_sobol_samples +from botorch.utils.test_helpers import get_pvar_expected +from botorch.utils.testing import _get_random_data, BotorchTestCase +from gpytorch import ExactMarginalLogLikelihood +from gpytorch.kernels import MaternKernel, ScaleKernel +from gpytorch.likelihoods import FixedNoiseGaussianLikelihood +from gpytorch.means import ConstantMean +from gpytorch.priors import GammaPrior + + +def _get_random_data_with_source(batch_shape, n, d, n_source, q=1, **tkwargs): + dtype = tkwargs.get("dtype", torch.float32) + rep_shape = batch_shape + torch.Size([1, 1]) + bounds = torch.stack([torch.zeros(d), torch.ones(d)]) + bounds[-1, -1] = n_source - 1 + train_x = ( + get_random_x_for_agp(n=n, bounds=bounds, q=q).repeat(rep_shape).type(dtype) + ) + train_y = torch.sin(train_x[..., :1] * (2 * math.pi)).type(dtype) + train_y = train_y + 0.2 * torch.randn(n, 1, **tkwargs).repeat(rep_shape) + return train_x, train_y + + +class TestAugmentedSingleTaskGP(BotorchTestCase): + def _get_model_and_data( + self, + batch_shape, + n, + d, + n_source, + outcome_transform=None, + input_transform=None, + extra_model_kwargs=None, + **tkwargs, + ): + extra_model_kwargs = extra_model_kwargs or {} + train_X, train_Y = _get_random_data_with_source( + batch_shape, n, d, n_source, **tkwargs + ) + model_kwargs = { + "train_X": train_X, + "train_Y": train_Y, + "outcome_transform": outcome_transform, + "input_transform": input_transform, + } + model = SingleTaskAugmentedGP(**model_kwargs, **extra_model_kwargs) + return model, model_kwargs + + def test_data_init(self): + d = 5 + for n, n_source in itertools.product((0, 1, 5, 10, 100), (1, 2, 5)): + bounds = torch.stack([torch.zeros(d), torch.ones(d)]) + bounds[-1, -1] = n_source - 1 + if n == 0: + self.assertRaises(InputDataError, get_random_x_for_agp, n, bounds, 1) + else: + x = get_random_x_for_agp(n, bounds, q=1) + self.assertIn(0, x[..., -1]) + self.assertEqual(x.shape, (n, d)) + + def test_init_error(self): + n, d = 10, 5 + for n_source, batch_shape in itertools.product( + (1, 2, 3), (torch.Size([]), torch.Size([2])) + ): + # Test initialization + train_X, train_Y = _get_random_data_with_source( + batch_shape=batch_shape, n=n, d=d, n_source=n_source + ) + if n_source == 1: + self.assertRaises( + InputDataError, SingleTaskAugmentedGP, train_X, train_Y + ) + continue + else: + model = SingleTaskAugmentedGP(train_X, train_Y) + self.assertIsInstance(model, SingleTaskAugmentedGP) + + # Test initialization without true source points + bounds = torch.stack([torch.zeros(d), torch.ones(d)]) + bounds[0, -1] = 1 + bounds[-1, -1] = n_source - 1 + train_X = draw_sobol_samples(bounds=bounds, n=n, q=1).squeeze(1) + train_X[:, -1] = torch.round(train_X[:, -1], decimals=0) + self.assertRaises(InputDataError, SingleTaskAugmentedGP, train_X, train_Y) + + def test_get_reliable_observation(self): + x = torch.linspace(0, 5, 15).reshape(-1, 1) + true_y = torch.sin(x).reshape(-1, 1) + y = torch.cos(x).reshape(-1, 1) + + model0 = SingleTaskGP(x, true_y) + model1 = SingleTaskGP(x, y) + + res = _get_reliable_observations(model0, model1, x) + true_res = torch.cat([torch.arange(0, 5, 1), torch.arange(9, 15, 1)]).int() + self.assertListEqual(res.tolist(), true_res.tolist()) + + def test_gp(self): + bounds = torch.tensor([[-1.0], [1.0]]) + d = 5 + for batch_shape, dtype, use_octf, use_intf in itertools.product( + (torch.Size(), torch.Size([2])), + (torch.float, torch.double), + (False, True), + (False, True), + ): + tkwargs = {"device": self.device, "dtype": dtype} + octf = Standardize(m=1, batch_shape=torch.Size()) if use_octf else None + intf = ( + Normalize(d=1, bounds=bounds.to(**tkwargs), transform_on_train=True) + if use_intf + else None + ) + model, model_kwargs = self._get_model_and_data( + batch_shape=batch_shape, + n=10, + d=d, + n_source=5, + outcome_transform=octf, + input_transform=intf, + **tkwargs, + ) + mll = ExactMarginalLogLikelihood(model.likelihood, model).to(**tkwargs) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=OptimizationWarning) + fit_gpytorch_mll( + mll, optimizer_kwargs={"options": {"maxiter": 1}}, max_attempts=1 + ) + + # test init + self.assertIsInstance(model.mean_module, ConstantMean) + self.assertIsInstance(model.covar_module, ScaleKernel) + matern_kernel = model.covar_module.base_kernel + self.assertIsInstance(matern_kernel, MaternKernel) + self.assertIsInstance(matern_kernel.lengthscale_prior, GammaPrior) + if use_octf: + self.assertIsInstance(model.outcome_transform, Standardize) + if use_intf: + self.assertIsInstance(model.input_transform, Normalize) + # permute output dim + train_X, train_Y, _ = model._transform_tensor_args( + X=model_kwargs["train_X"], Y=model_kwargs["train_Y"] + ) + # check that the train inputs have been transformed and set + # on the model for each source + for s in train_X[..., -1].unique(): + self.assertTrue( + torch.equal( + model.models[int(s)].train_inputs[0], + intf(train_X[train_X[..., -1] == s][..., :-1]), + ) + ) + + # test posterior + # test non batch evaluation + X = torch.rand(batch_shape + torch.Size([3, d - 1]), **tkwargs) + expected_shape = batch_shape + torch.Size([3, 1]) + posterior = model.posterior(X) + self.assertIsInstance(posterior, GPyTorchPosterior) + self.assertEqual(posterior.mean.shape, expected_shape) + self.assertEqual(posterior.variance.shape, expected_shape) + + # test adding observation noise + posterior_pred = model.posterior(X, observation_noise=True) + self.assertIsInstance(posterior_pred, GPyTorchPosterior) + self.assertEqual(posterior_pred.mean.shape, expected_shape) + self.assertEqual(posterior_pred.variance.shape, expected_shape) + if use_octf: + # ensure un-transformation is applied + tmp_tf = model.outcome_transform + del model.outcome_transform + pp_tf = model.posterior(X, observation_noise=True) + model.outcome_transform = tmp_tf + expected_var = tmp_tf.untransform_posterior(pp_tf).variance + self.assertAllClose(posterior_pred.variance, expected_var) + else: + pvar = posterior_pred.variance + pvar_exp = get_pvar_expected(posterior, model, X, 1) + self.assertAllClose(pvar, pvar_exp, rtol=1e-4, atol=1e-5) + + # Tensor valued observation noise. + obs_noise = torch.rand((*X.shape[:-1], 1), **tkwargs) + posterior_pred = model.posterior(X, observation_noise=obs_noise) + self.assertIsInstance(posterior_pred, GPyTorchPosterior) + self.assertEqual(posterior_pred.mean.shape, expected_shape) + self.assertEqual(posterior_pred.variance.shape, expected_shape) + if use_octf: + _, obs_noise = model.outcome_transform.untransform(obs_noise, obs_noise) + self.assertAllClose(posterior_pred.variance, posterior.variance + obs_noise) + + def test_condition_on_observations(self): + for dtype, use_octf in itertools.product( + (torch.float, torch.double), + (False, True), + ): + d = 5 + tkwargs = {"device": self.device, "dtype": dtype} + octf = Standardize(m=1) if use_octf else None + model, model_kwargs = self._get_model_and_data( + batch_shape=torch.Size([]), + n=10, + d=d, + n_source=5, + outcome_transform=octf, + **tkwargs, + ) + d = d - 1 + # evaluate model + model.posterior(torch.rand(torch.Size([4, d]), **tkwargs)) + # test condition_on_observations + fant_shape = torch.Size([2]) + # fantasize at different input points + X_fant, Y_fant = _get_random_data( + batch_shape=fant_shape + torch.Size([]), m=1, d=d, n=3, **tkwargs + ) + c_kwargs = ( + {"noise": torch.full_like(Y_fant, 0.01)} + if isinstance(model, FixedNoiseGP) + else {} + ) + cm = model.condition_on_observations(X_fant, Y_fant, **c_kwargs) + # fantasize at same input points (check proper broadcasting) + c_kwargs_same_inputs = ( + {"noise": torch.full_like(Y_fant[0], 0.01)} + if isinstance(model, FixedNoiseGP) + else {} + ) + cm_same_inputs = model.condition_on_observations( + X_fant[0], Y_fant, **c_kwargs_same_inputs + ) + + test_Xs = [ + # test broadcasting single input across fantasy and model batches + torch.rand(4, d, **tkwargs), + # separate input for each model batch and broadcast across + # fantasy batches + torch.rand(torch.Size([]) + torch.Size([4, d]), **tkwargs), + # separate input for each model and fantasy batch + torch.rand(fant_shape + torch.Size([]) + torch.Size([4, d]), **tkwargs), + ] + for test_X in test_Xs: + posterior = cm.posterior(test_X) + self.assertEqual( + posterior.mean.shape, + fant_shape + torch.Size([]) + torch.Size([4, 1]), + ) + posterior_same_inputs = cm_same_inputs.posterior(test_X) + self.assertEqual( + posterior_same_inputs.mean.shape, + fant_shape + torch.Size([]) + torch.Size([4, 1]), + ) + + # check that fantasies of batched model are correct + if len(torch.Size([])) > 0 and test_X.dim() == 2: + state_dict_non_batch = { + key: (val[0] if val.numel() > 1 else val) + for key, val in model.state_dict().items() + } + model_kwargs_non_batch = { + "train_X": model_kwargs["train_X"][0], + "train_Y": model_kwargs["train_Y"][0], + } + if "train_Yvar" in model_kwargs: + model_kwargs_non_batch["train_Yvar"] = model_kwargs[ + "train_Yvar" + ][0] + if model_kwargs["outcome_transform"] is not None: + model_kwargs_non_batch["outcome_transform"] = Standardize(m=1) + model_non_batch = type(model)(**model_kwargs_non_batch) + model_non_batch.load_state_dict(state_dict_non_batch) + model_non_batch.eval() + model_non_batch.likelihood.eval() + model_non_batch.posterior(torch.rand(torch.Size([4, 1]), **tkwargs)) + c_kwargs = ( + {"noise": torch.full_like(Y_fant[0, 0, :], 0.01)} + if isinstance(model, FixedNoiseGP) + else {} + ) + cm_non_batch = model_non_batch.condition_on_observations( + X_fant[0][0], Y_fant[:, 0, :], **c_kwargs + ) + non_batch_posterior = cm_non_batch.posterior(test_X) + self.assertTrue( + torch.allclose( + posterior_same_inputs.mean[:, 0, ...], + non_batch_posterior.mean, + atol=1e-3, + ) + ) + self.assertTrue( + torch.allclose( + posterior_same_inputs.distribution.covariance_matrix[ + :, 0, :, : + ], + non_batch_posterior.distribution.covariance_matrix, + atol=1e-3, + ) + ) + + +class TestAugmentedFixedNoiseGP(TestAugmentedSingleTaskGP): + def _get_model_and_data( + self, + batch_shape, + n, + d, + n_source, + outcome_transform=None, + input_transform=None, + extra_model_kwargs=None, + **tkwargs, + ): + extra_model_kwargs = extra_model_kwargs or {} + train_X, train_Y = _get_random_data_with_source( + batch_shape, n, d, n_source, **tkwargs + ) + model_kwargs = { + "train_X": train_X, + "train_Y": train_Y, + "train_Yvar": torch.full_like(train_Y, 0.01), + "outcome_transform": outcome_transform, + "input_transform": input_transform, + } + model = FixedNoiseAugmentedGP(**model_kwargs, **extra_model_kwargs) + return model, model_kwargs + + def test_init_error(self): + n, d = 10, 5 + for n_source, batch_shape in itertools.product( + (1, 2, 3), (torch.Size([]), torch.Size([2])) + ): + # Test initialization + train_X, train_Y = _get_random_data_with_source( + batch_shape=batch_shape, n=n, d=d, n_source=n_source + ) + if n_source == 1: + self.assertRaises( + InputDataError, + FixedNoiseAugmentedGP, + train_X, + train_Y, + torch.full_like(train_Y, 0.01), + ) + continue + else: + model = FixedNoiseAugmentedGP( + train_X, train_Y, torch.full_like(train_Y, 0.01) + ) + self.assertIsInstance(model, FixedNoiseAugmentedGP) + + # Test initialization without true source points + bounds = torch.stack([torch.zeros(d), torch.ones(d)]) + bounds[0, -1] = 1 + bounds[-1, -1] = n_source - 1 + train_X = draw_sobol_samples(bounds=bounds, n=n, q=1).squeeze(1) + train_X[:, -1] = torch.round(train_X[:, -1], decimals=0) + self.assertRaises( + InputDataError, + FixedNoiseAugmentedGP, + train_X, + train_Y, + torch.full_like(train_Y, 0.01), + ) + + def test_get_reliable_observation(self): + x = torch.linspace(0, 5, 15).reshape(-1, 1) + true_y = torch.sin(x).reshape(-1, 1) + y = torch.cos(x).reshape(-1, 1) + + model0 = FixedNoiseGP(x, true_y, torch.full_like(true_y, 1)) + model1 = FixedNoiseGP(x, y, torch.full_like(true_y, 1)) + + res = _get_reliable_observations(model0, model1, x) + true_res = torch.cat([torch.arange(0, 4, 1), torch.arange(10, 13, 1)]).int() + self.assertListEqual(res.tolist(), true_res.tolist()) + + def test_fixed_noise_likelihood(self): + for batch_shape, dtype in itertools.product( + (torch.Size(), torch.Size([2])), (torch.float, torch.double) + ): + tkwargs = {"device": self.device, "dtype": dtype} + model, model_kwargs = self._get_model_and_data( + batch_shape=batch_shape, + n=10, + d=5, + n_source=5, + **tkwargs, + ) + self.assertIsInstance(model.likelihood, FixedNoiseGaussianLikelihood) + likelihood_noise = model.likelihood.noise.contiguous().view(-1) + train_Y_var = model_kwargs["train_Yvar"].contiguous().view(-1) + self.assertTrue( + torch.equal( + likelihood_noise, + train_Y_var[: len(likelihood_noise)], + ) + ) + + def test_fantasized_noise(self): + for batch_shape, dtype, use_octf in itertools.product( + (torch.Size(), torch.Size([2])), + (torch.float, torch.double), + (False, True), + ): + d = 5 + tkwargs = {"device": self.device, "dtype": dtype} + octf = Standardize(m=1, batch_shape=torch.Size()) if use_octf else None + model, _ = self._get_model_and_data( + batch_shape=batch_shape, + n=10, + d=d, + n_source=5, + outcome_transform=octf, + **tkwargs, + ) + # fantasize + X_f = torch.rand( + torch.Size(batch_shape + torch.Size([4, d - 1])), **tkwargs + ) + sampler = SobolQMCNormalSampler(sample_shape=torch.Size([3])) + fm = model.fantasize(X=X_f, sampler=sampler) + noise = model.likelihood.noise.unsqueeze(-1) + avg_noise = noise.mean(dim=-2, keepdim=True) + fm_noise = fm.likelihood.noise.unsqueeze(-1) + + self.assertTrue((fm_noise[..., -4:, :] == avg_noise).all()) + # pass tensor of noise + # noise is assumed to be outcome transformed + # batch shape x n' x m + obs_noise = torch.full( + X_f.shape[:-1] + torch.Size([1]), 0.1, dtype=dtype, device=self.device + ) + fm = model.fantasize(X=X_f, sampler=sampler, observation_noise=obs_noise) + fm_noise = fm.likelihood.noise.unsqueeze(-1) + self.assertTrue((fm_noise[..., -4:, :] == obs_noise).all()) + # test batch shape x 1 x m + obs_noise = torch.full( + X_f.shape[:-2] + torch.Size([1, 1]), + 0.1, + dtype=dtype, + device=self.device, + ) + fm = model.fantasize(X=X_f, sampler=sampler, observation_noise=obs_noise) + fm_noise = fm.likelihood.noise.unsqueeze(-1) + self.assertTrue( + ( + fm_noise[..., -4:, :] + == obs_noise.expand(X_f.shape[:-1] + torch.Size([1])) + ).all() + ) From 85bbb6ac347b426a77db3ace72402a73db81bca0 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 11:18:33 +0100 Subject: [PATCH 04/20] Add AUCB unit test --- test/acquisition/test_multi_source.py | 147 ++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 test/acquisition/test_multi_source.py diff --git a/test/acquisition/test_multi_source.py b/test/acquisition/test_multi_source.py new file mode 100644 index 0000000000..23b106bc3e --- /dev/null +++ b/test/acquisition/test_multi_source.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import torch + +from botorch.acquisition.augmented_multi_source import AugmentedUpperConfidenceBound +from botorch.exceptions import UnsupportedError +from botorch.models.gp_regression_multisource import SingleTaskAugmentedGP +from botorch.utils.testing import BotorchTestCase, MockModel, MockPosterior + + +class TestAugmentedUpperConfidenceBound(BotorchTestCase): + def _get_mock_agp(self, batch_shape, dtype): + train_X = torch.tensor([[0, 0], [0, 1]], dtype=dtype, device=self.device) + train_Y = torch.tensor([[0.5], [5.0]], dtype=dtype, device=self.device) + rep_shape = batch_shape + torch.Size([1, 1]) + train_X = train_X.repeat(rep_shape) + train_Y = train_Y.repeat(rep_shape) + model_kwargs = { + "train_X": train_X, + "train_Y": train_Y, + } + model = SingleTaskAugmentedGP(**model_kwargs) + return model + + def test_upper_confidence_bound(self): + for dtype in (torch.float, torch.double): + mm = self._get_mock_agp(torch.Size([]), dtype) + module = AugmentedUpperConfidenceBound( + model=mm, + beta=1.0, + best_f=torch.tensor(0.5, device=self.device, dtype=dtype), + cost={0: 1, 1: 0.5}, + ) + X = torch.zeros(1, 2, device=self.device, dtype=dtype) + ucb = module(X) + ucb_expected = torch.tensor([1.8460], device=self.device, dtype=dtype) + self.assertAllClose(ucb, ucb_expected, atol=1e-4) + + module = AugmentedUpperConfidenceBound( + model=mm, + beta=1.0, + maximize=False, + best_f=torch.tensor(0.5, device=self.device, dtype=dtype), + cost={0: 1, 1: 0.5}, + ) + X = torch.zeros(1, 2, device=self.device, dtype=dtype) + ucb = module(X) + ucb_expected = torch.tensor([0.1217], device=self.device, dtype=dtype) + self.assertAllClose(ucb, ucb_expected, atol=1e-4) + + # check for proper error if not multi-source model + mean = torch.rand(1, 1, device=self.device, dtype=dtype) + variance = torch.rand(1, 1, device=self.device, dtype=dtype) + mm1 = MockModel(MockPosterior(mean=mean, variance=variance)) + with self.assertRaises(UnsupportedError): + AugmentedUpperConfidenceBound( + model=mm1, + beta=1.0, + best_f=torch.tensor(1.0, device=self.device, dtype=dtype), + cost={0: 1, 1: 0.5}, + ) + # check for proper error if multi-output model + mean2 = torch.rand(1, 2, device=self.device, dtype=dtype) + variance2 = torch.rand(1, 2, device=self.device, dtype=dtype) + mm2 = MockModel(MockPosterior(mean=mean2, variance=variance2)) + mm2.models = [] + with self.assertRaises(UnsupportedError): + AugmentedUpperConfidenceBound( + model=mm2, + beta=1.0, + best_f=torch.tensor(1.0, device=self.device, dtype=dtype), + cost={0: 1, 1: 0.5}, + ) + + def test_upper_confidence_bound_batch(self): + for dtype in (torch.float, torch.double): + mm = self._get_mock_agp(torch.Size([2]), dtype) + module = AugmentedUpperConfidenceBound( + model=mm, + beta=1.0, + best_f=torch.tensor(1.0, device=self.device, dtype=dtype), + cost={0: 1, 1: 0.5}, + ) + X = torch.zeros(1, 2, device=self.device, dtype=dtype) + ucb = module(X) + ucb_expected = torch.tensor([2.3892], device=self.device, dtype=dtype) + self.assertAllClose(ucb, ucb_expected, atol=1e-4) + + # check for proper error if not multi-source model + mean = torch.rand(3, 1, 1, device=self.device, dtype=dtype) + variance = torch.rand(3, 1, 1, device=self.device, dtype=dtype) + mm1 = MockModel(MockPosterior(mean=mean, variance=variance)) + with self.assertRaises(UnsupportedError): + AugmentedUpperConfidenceBound( + model=mm1, + beta=1.0, + best_f=torch.tensor(1.0, device=self.device, dtype=dtype), + cost={0: 1, 1: 0.5}, + ) + # check for proper error if multi-output model + mean2 = torch.rand(3, 1, 2, device=self.device, dtype=dtype) + variance2 = torch.rand(3, 1, 2, device=self.device, dtype=dtype) + mm2 = MockModel(MockPosterior(mean=mean2, variance=variance2)) + mm2.models = [] + with self.assertRaises(UnsupportedError): + AugmentedUpperConfidenceBound( + model=mm2, + beta=1.0, + best_f=torch.tensor(1.0, device=self.device, dtype=dtype), + cost={0: 1, 1: 0.5}, + ) + + def test_get_mean_and_sigma(self): + for dtype in (torch.float, torch.double): + # Test with overall model + mean = torch.rand(1, 1, device=self.device, dtype=dtype) + variance = torch.rand(1, 1, device=self.device, dtype=dtype) + mm = MockModel(MockPosterior(mean=mean, variance=variance)) + mm.models = [] + module = AugmentedUpperConfidenceBound( + model=mm, + beta=1.0, + best_f=torch.tensor(1.0, device=self.device, dtype=dtype), + cost={0: 1, 1: 0.5}, + ) + X = torch.zeros(1, 2, device=self.device, dtype=dtype) + mm_mean, mm_sigma = module._mean_and_sigma(X) + self.assertAllClose(mm_mean, mean.squeeze(-1).squeeze(-1), atol=1e-4) + self.assertAllClose( + torch.pow(mm_sigma, 2), variance.squeeze(-1).squeeze(-1), atol=1e-4 + ) + _, mm_sigma = module._mean_and_sigma(X, compute_sigma=False) + self.assertIsNone(mm_sigma) + # Test with specific model + mean2 = torch.rand(1, 1, device=self.device, dtype=dtype) + variance2 = torch.rand(1, 1, device=self.device, dtype=dtype) + mm2 = MockModel(MockPosterior(mean=mean2, variance=variance2)) + X = torch.zeros(1, 2, device=self.device, dtype=dtype) + mm_mean, mm_sigma = module._mean_and_sigma(X, mm2) + self.assertAllClose(mm_mean, mean2.squeeze(-1).squeeze(-1), atol=1e-4) + self.assertAllClose( + torch.pow(mm_sigma, 2), variance2.squeeze(-1).squeeze(-1), atol=1e-4 + ) From 416e8d93c6338d394ea0e63a6390111e3f0f0b40 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 12:35:54 +0100 Subject: [PATCH 05/20] Doc refactor --- ...lti_source.py => augmented_multisource.py} | 26 +++++++++---------- botorch/models/gp_regression_multisource.py | 9 +++++-- sphinx/source/acquisition.rst | 5 ++++ sphinx/source/models.rst | 5 ++++ test/acquisition/test_multi_source.py | 2 +- 5 files changed, 31 insertions(+), 16 deletions(-) rename botorch/acquisition/{augmented_multi_source.py => augmented_multisource.py} (84%) diff --git a/botorch/acquisition/augmented_multi_source.py b/botorch/acquisition/augmented_multisource.py similarity index 84% rename from botorch/acquisition/augmented_multi_source.py rename to botorch/acquisition/augmented_multisource.py index 827c02f90d..ee1dbc3fe7 100644 --- a/botorch/acquisition/augmented_multi_source.py +++ b/botorch/acquisition/augmented_multisource.py @@ -22,19 +22,16 @@ class AugmentedUpperConfidenceBound(UpperConfidenceBound): r"""Single-outcome Multi-Source Upper Confidence Bound (UCB). - Description... - - `AUCB(x, s, y^+) = - ((mu(x) + sqrt(beta) * sigma(x)) - y^+) / - (c(s) (1 + abs(mu(x) - mu_s(x))))`, - where `mu` and `sigma` are the posterior mean and standard deviation of the AGP, - `mu_s` is the posterior mean of the GP modelling the s-th source and - c(s) is the cost of the source s. - - Example: - >>> model = AugmentedSingleTaskGP(train_X, train_Y) - >>> UCB = AugmentedUpperConfidenceBound(model, beta=0.2) - >>> ucb = UCB(test_X) + A modified version of the UCB for Multi Information Source, that consider + the most optimistic improvement with respect to the best value observed so far. + The improvement is then penalized depending on source’s cost, and + the discrepancy between the GP associated to the source and the AGP. + + `AUCB(x, s, y^+) = ((mu(x) + sqrt(beta) * sigma(x)) - y^+) + / (c(s) (1 + abs(mu(x) - mu_s(x))))`, + where `mu` and `sigma` are the posterior mean and standard deviation of the AGP, + `mu_s` is the posterior mean of the GP modelling the s-th source and + c(s) is the cost of the source s. """ def __init__( @@ -52,6 +49,9 @@ def __init__( model: A fitted single-outcome Augmented GP model. beta: Either a scalar or a one-dim tensor with `b` elements (batch mode) representing the trade-off parameter between mean and covariance + cost: A dictionary containing the cost of querying each source. + best_f: Either a scalar or a `b`-dim Tensor (batch mode) representing + the best function value observed so far (assumed noiseless). posterior_transform: A PosteriorTransform. If using a multi-output model, a PosteriorTransform that transforms the multi-output posterior into a single-output posterior is required. diff --git a/botorch/models/gp_regression_multisource.py b/botorch/models/gp_regression_multisource.py index c8f5797b22..09fb3c0524 100644 --- a/botorch/models/gp_regression_multisource.py +++ b/botorch/models/gp_regression_multisource.py @@ -10,8 +10,6 @@ For more on Multi-Source BO, see the `tutorial `__ -BRIEF DESCRIPTION - .. [Ca2021ms] Candelieri, A., & Archetti, F. (2021). Sparsifying to optimize over multiple information sources: @@ -78,6 +76,13 @@ class SingleTaskAugmentedGP(SingleTaskGP): r"""A single-task multi-source GP model. The Augmented Gaussian Process is described in [Ca2021ms]_. + The basic idea is to use GP sparsification for selecting a subset of + the function evaluations, among those performed so far over all the + different sources, as inducing locations to generate the AGP approximating f(x). + The GP sparsification proposed is an insertion method: the set of inducing + locations is initialized with the function evaluations on the most expensive + information source and is incremented by including evaluations on other sources + depending on both a model discrepancy measure and GP’s predictive uncertainty. """ def __init__( diff --git a/sphinx/source/acquisition.rst b/sphinx/source/acquisition.rst index 733f7eb60a..77aafef6b9 100644 --- a/sphinx/source/acquisition.rst +++ b/sphinx/source/acquisition.rst @@ -143,6 +143,11 @@ Preference Acquisition Functions .. automodule:: botorch.acquisition.preference :members: +Multi Information Source Acquisition Functions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: botorch.acquisition.augmented_multisource + :members: + Objectives and Cost-Aware Utilities ------------------------------------------- diff --git a/sphinx/source/models.rst b/sphinx/source/models.rst index c8a6f8fe55..6f5ffcac7a 100644 --- a/sphinx/source/models.rst +++ b/sphinx/source/models.rst @@ -44,6 +44,11 @@ GP Regression Models .. automodule:: botorch.models.gp_regression :members: +Multi Information Source GP Regression Models +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. automodule:: botorch.models.gp_regression_multisource + :members: + Multi-Fidelity GP Regression Models ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: botorch.models.gp_regression_fidelity diff --git a/test/acquisition/test_multi_source.py b/test/acquisition/test_multi_source.py index 23b106bc3e..00636de4a8 100644 --- a/test/acquisition/test_multi_source.py +++ b/test/acquisition/test_multi_source.py @@ -6,7 +6,7 @@ import torch -from botorch.acquisition.augmented_multi_source import AugmentedUpperConfidenceBound +from botorch.acquisition.augmented_multisource import AugmentedUpperConfidenceBound from botorch.exceptions import UnsupportedError from botorch.models.gp_regression_multisource import SingleTaskAugmentedGP from botorch.utils.testing import BotorchTestCase, MockModel, MockPosterior From 3055268ec130477298d7ad54d2ff0119577c7e9b Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 12:45:36 +0100 Subject: [PATCH 06/20] Add tutorial notebook for AGP --- tutorials/multi_source_bo.ipynb | 704 ++++++++++++++++++++++++++++++++ 1 file changed, 704 insertions(+) create mode 100644 tutorials/multi_source_bo.ipynb diff --git a/tutorials/multi_source_bo.ipynb b/tutorials/multi_source_bo.ipynb new file mode 100644 index 0000000000..93186bcf88 --- /dev/null +++ b/tutorials/multi_source_bo.ipynb @@ -0,0 +1,704 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "## Multi-Information Source BO with Augmented Gaussian Processes\n", + "\n", + "In this tutorial, we show how to perform multi-information source Bayesian optimization in BoTorch using the Augmented Gaussian Process (AGP) along with the Augmented UCB (AUCB) acquisition function [1]. The key idea of AGP is to fit a GP model for each information source and *augment* the GP of the high fidelity source with observations from the lower fidelities sources. Only observations considered as *reliable* are used to augment the GP. The UCB acquisition function is modified to take into account also the sources' cost.\n", + "\n", + "We find the AGP performs well if compared with discrete multi-fidelity approaches [2].\n", + "\n", + "[1] [Candelieri, A., & Archetti, F. (2021). Sparsifying to optimize over multiple information sources: an augmented Gaussian process based algorithm. Structural and Multidisciplinary Optimization, 64, 239-255.](https://link.springer.com/article/10.1007/s00158-021-02882-7)\n", + "[2] [The arxiv will be available soon.](https://arxiv.org/)\n" + ], + "metadata": { + "collapsed": false + }, + "id": "826ca0dcff42a3ba" + }, + { + "cell_type": "code", + "execution_count": 1, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: matplotlib in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (3.8.2)\n", + "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (1.2.0)\n", + "Requirement already satisfied: cycler>=0.10 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (4.46.0)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (1.4.5)\n", + "Requirement already satisfied: numpy<2,>=1.21 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (1.26.0)\n", + "Requirement already satisfied: packaging>=20.0 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (23.2)\n", + "Requirement already satisfied: pillow>=8 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (10.1.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (3.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (2.8.2)\n", + "Requirement already satisfied: six>=1.5 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[notice] A new release of pip is available: 23.2.1 -> 23.3.2\n", + "[notice] To update, run: python.exe -m pip install --upgrade pip\n" + ] + } + ], + "source": [ + "!pip install matplotlib" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:37:32.213316900Z", + "start_time": "2023-12-18T11:37:29.151143400Z" + } + }, + "id": "8aa9032dbb2c2b04" + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [], + "source": [ + "import os\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "from gpytorch import ExactMarginalLogLikelihood\n", + "\n", + "import botorch\n", + "from botorch import fit_gpytorch_mll\n", + "from botorch.acquisition import InverseCostWeightedUtility, qMultiFidelityMaxValueEntropy\n", + "from botorch.acquisition.augmented_multisource import AugmentedUpperConfidenceBound\n", + "from botorch.models import AffineFidelityCostModel, SingleTaskMultiFidelityGP\n", + "from botorch.models.gp_regression_multisource import SingleTaskAugmentedGP, get_random_x_for_agp\n", + "from botorch.models.transforms import Standardize\n", + "from botorch.optim import optimize_acqf, optimize_acqf_mixed\n", + "from botorch.test_functions.multi_fidelity import AugmentedBranin" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:37:36.050798200Z", + "start_time": "2023-12-18T11:37:32.214312700Z" + } + }, + "id": "e55defd1ee4a5b0f" + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "initial_id", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2023-12-18T11:37:36.056871Z", + "start_time": "2023-12-18T11:37:36.054311400Z" + } + }, + "outputs": [], + "source": [ + "tkwargs = {\n", + " \"dtype\": torch.double,\n", + " \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n", + "}\n", + "SMOKE_TEST = os.environ.get(\"SMOKE_TEST\", False)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "outputs": [], + "source": [ + "N_ITER = 10 if SMOKE_TEST else 50\n", + "SEED = 3" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:37:36.071646400Z", + "start_time": "2023-12-18T11:37:36.057904700Z" + } + }, + "id": "e316bd291459a135" + }, + { + "cell_type": "markdown", + "source": [ + "### Problem setup\n", + "We'll consider the Augmented Branin multi-fidelity synthetic test problem. This function is a version of the Branin test function with an additional dimension representing the fidelity parameter. The function takes the form $f(x,s)$ where $x \\in [-5, 10] \\times [0, 15]$ and $s \\in [0,1]$. The target fidelity is 1.0, which means that our goal is to solve $\\max_x f(x,1.0)$ by making use of cheaper evaluations $f(x,s)$ for $s < 1.0$. In this example, we'll assume that the cost function takes the form $5.0 + s$, illustrating a situation where the fixed cost is $5.0$.\n", + "\n", + "Since a multi-information source context is considered, three different sources we'll be used with $s = 0.5, 0.75, 1.00$ respectively." + ], + "metadata": { + "collapsed": false + }, + "id": "6b58c67f5dbf329c" + }, + { + "cell_type": "code", + "execution_count": 5, + "outputs": [], + "source": [ + "problem = AugmentedBranin(negate=True).to(**tkwargs)\n", + "fidelities = torch.tensor([0.5, 0.75, 1.0], **tkwargs)\n", + "n_sources = fidelities.shape[0]\n", + "\n", + "bounds = torch.tensor([[-5, 0, 0], [10, 15, n_sources - 1]], **tkwargs)\n", + "target_fidelities = {n_sources - 1: 1.0}\n", + "\n", + "cost_model = AffineFidelityCostModel(fidelity_weights=target_fidelities, fixed_cost=5.0)\n", + "cost_aware_utility = InverseCostWeightedUtility(cost_model=cost_model)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:37:36.085138100Z", + "start_time": "2023-12-18T11:37:36.071646400Z" + } + }, + "id": "5f13380e681011ea" + }, + { + "cell_type": "markdown", + "source": [ + "### Model initialization\n", + "\n", + "We use a `SingleTaskAugmentedGP` to implement our AGP.\n", + "\n", + "At each Bayesian Optimization iteration, the set of observations from the *ground-truth* (i.e., the highest fidelity and more expensive source) is temporarily *augmented* by including observations from the other cheap sources, only if they can be considered *reliable*. Specifically, an observation $(x,y)$ from a cheap source is considered reliable if it satisfies the following inequality:\n", + "\n", + "$$\\vert\\mu(x)-y\\vert \\leq m \\sigma(x)$$\n", + "\n", + "where $\\mu(x)$ and $\\sigma(x)$ are, respectively, the posterior mean and standard deviation of the GP model fitted on the high fidelity observations only, and $m$ is a technical parameter making more *conservative* ($m→0$) or *inclusive* ($m→∞)$ the augmentation process. As reported in [1], a suitable value for this parameter is $m=1$.\n", + "\n", + "After the set of observations is augmented, the AGP is fitted through `SingleTaskAugmentedGP`.\n" + ], + "metadata": { + "collapsed": false + }, + "id": "81e30344694a9583" + }, + { + "cell_type": "code", + "execution_count": 6, + "outputs": [], + "source": [ + "def generate_initial_data(n):\n", + " train_x = get_random_x_for_agp(n, bounds, 1)\n", + " xs = train_x[..., :-1]\n", + " fids = fidelities[train_x[..., -1].int()].reshape(-1, 1)\n", + " train_obj = problem(torch.cat((xs, fids), dim=1)).unsqueeze(-1)\n", + " return train_x, train_obj\n", + "\n", + "\n", + "def initialize_model(train_x, train_obj, m):\n", + " model = SingleTaskAugmentedGP(\n", + " train_x, train_obj, m=m, outcome_transform=Standardize(m=1),\n", + " )\n", + " mll = ExactMarginalLogLikelihood(model.likelihood, model)\n", + " return mll, model" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:37:36.086286100Z", + "start_time": "2023-12-18T11:37:36.077862900Z" + } + }, + "id": "f8272160f69227ef" + }, + { + "cell_type": "markdown", + "source": [ + "#### Define a helper function that performs the essential BO step\n", + "This helper function optimizes the acquisition function and returns the candidate point along with the observed function values.\n", + "\n", + "The UCB acquisition function has been modified to deal with both the *discrepancy* between information sources and the *source-specific query cost*.\n", + "\n", + "Formally, the AUCB acquisition function, at a generic iteration $t$, is defined as:\n", + "\n", + "$$\\alpha_s(x,\\hat y^+) = \\frac{\\left[\\hat{\\mu}(x) + \\sqrt{\\beta^{(t)}} \\hat{\\sigma}(x)\\right] - \\hat{y}^+}{c_s \\cdot (1+\\vert \\hat{\\mu}(x) - \\mu_s(x) \\vert)} $$\n", + "\n", + "where $\\hat{y}^+$ is the best (i.e., highest) value in the *augmented* set of observations, the numerator is -- therefore -- the optimistic improvement with respect to $\\hat{y}^+$, $c_s$ is the query cost for the source $s$, and $\\vert \\hat{\\mu}(x) - \\mu_s(x) \\vert$ is a discrepancy measure between the predictions provided by the AGP and the GP on the source $s$, respectively, given the input $x$ (i.e., 1 is added just to avoid division by zero).\n", + "\n", + "For more information, please refer to [1]," + ], + "metadata": { + "collapsed": false + }, + "id": "ad21cd7999805d78" + }, + { + "cell_type": "code", + "execution_count": 7, + "outputs": [], + "source": [ + "def optimize_aucb(acqf):\n", + " candidate, value = optimize_acqf(\n", + " acq_function=acqf,\n", + " bounds=bounds,\n", + " q=1,\n", + " num_restarts=5,\n", + " raw_samples=128,\n", + " )\n", + " # observe new values\n", + " new_x = candidate.detach()\n", + " new_x[:, -1] = torch.round(new_x[:, -1], decimals=0)\n", + " return new_x" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:37:36.088896700Z", + "start_time": "2023-12-18T11:37:36.085138100Z" + } + }, + "id": "311309bd6a4d3a92" + }, + { + "cell_type": "markdown", + "source": [ + "### Perform a few steps of multi-fidelity BO\n", + "First, let's generate some initial random data and fit a surrogate model." + ], + "metadata": { + "collapsed": false + }, + "id": "667b55ca7ae58af3" + }, + { + "cell_type": "code", + "execution_count": 8, + "outputs": [], + "source": [ + "torch.manual_seed(SEED)\n", + "train_x, train_obj = generate_initial_data(n=5)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:37:36.137343Z", + "start_time": "2023-12-18T11:37:36.089845700Z" + } + }, + "id": "a2b9dbfbae2f7d5" + }, + { + "cell_type": "markdown", + "source": [ + "We can now use the helper functions above to run a few iterations of BO." + ], + "metadata": { + "collapsed": false + }, + "id": "ca54230c1481ba60" + }, + { + "cell_type": "code", + "execution_count": 9, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iter 0;\t Fid = 1.00;\t Obj = -12.0999;\n", + "Iter 1;\t Fid = 1.00;\t Obj = -50.0743;\n", + "Iter 2;\t Fid = 0.50;\t Obj = -11.9672;\n", + "Iter 3;\t Fid = 0.50;\t Obj = -8.7046;\n", + "Iter 4;\t Fid = 0.50;\t Obj = -13.3425;\n", + "Iter 5;\t Fid = 1.00;\t Obj = -11.6850;\n", + "Iter 6;\t Fid = 0.50;\t Obj = -8.2926;\n", + "Iter 7;\t Fid = 1.00;\t Obj = -14.4455;\n", + "Iter 8;\t Fid = 0.50;\t Obj = -16.5712;\n", + "Iter 9;\t Fid = 1.00;\t Obj = -32.4978;\n", + "Iter 10;\t Fid = 0.50;\t Obj = -17.8776;\n", + "Iter 11;\t Fid = 1.00;\t Obj = -17.4798;\n", + "Iter 12;\t Fid = 0.50;\t Obj = -14.4403;\n", + "Iter 13;\t Fid = 1.00;\t Obj = -16.5080;\n", + "Iter 14;\t Fid = 0.50;\t Obj = -48.2065;\n", + "Iter 15;\t Fid = 1.00;\t Obj = -67.0580;\n", + "Iter 16;\t Fid = 0.50;\t Obj = -113.7052;\n", + "Iter 17;\t Fid = 1.00;\t Obj = -5.5352;\n", + "Iter 18;\t Fid = 1.00;\t Obj = -14.0937;\n", + "Iter 19;\t Fid = 0.50;\t Obj = -116.5878;\n", + "Iter 20;\t Fid = 1.00;\t Obj = -5.2398;\n", + "Iter 21;\t Fid = 1.00;\t Obj = -2.9797;\n", + "Iter 22;\t Fid = 1.00;\t Obj = -0.9191;\n", + "Iter 23;\t Fid = 1.00;\t Obj = -0.4690;\n", + "Iter 24;\t Fid = 1.00;\t Obj = -2.7735;\n", + "Iter 25;\t Fid = 1.00;\t Obj = -5.6576;\n", + "Iter 26;\t Fid = 1.00;\t Obj = -0.4424;\n", + "Iter 27;\t Fid = 0.50;\t Obj = -144.6051;\n", + "Iter 28;\t Fid = 1.00;\t Obj = -2.1584;\n", + "Iter 29;\t Fid = 1.00;\t Obj = -37.6281;\n", + "Iter 30;\t Fid = 1.00;\t Obj = -2.3379;\n", + "Iter 31;\t Fid = 0.50;\t Obj = -230.5202;\n", + "Iter 32;\t Fid = 0.75;\t Obj = -77.1763;\n", + "Iter 33;\t Fid = 1.00;\t Obj = -3.1912;\n", + "Iter 34;\t Fid = 1.00;\t Obj = -1.5353;\n", + "Iter 35;\t Fid = 1.00;\t Obj = -1.2452;\n", + "Iter 36;\t Fid = 1.00;\t Obj = -1.4735;\n", + "Iter 37;\t Fid = 1.00;\t Obj = -3.5458;\n", + "Iter 38;\t Fid = 0.75;\t Obj = -58.2390;\n", + "Iter 39;\t Fid = 1.00;\t Obj = -4.2493;\n", + "Iter 40;\t Fid = 0.50;\t Obj = -53.5646;\n", + "Iter 41;\t Fid = 0.50;\t Obj = -17.8283;\n", + "Iter 42;\t Fid = 1.00;\t Obj = -21.7001;\n", + "Iter 43;\t Fid = 1.00;\t Obj = -3.9768;\n", + "Iter 44;\t Fid = 1.00;\t Obj = -10.2322;\n", + "Iter 45;\t Fid = 1.00;\t Obj = -1.1175;\n", + "Iter 46;\t Fid = 1.00;\t Obj = -3.9205;\n", + "Iter 47;\t Fid = 0.75;\t Obj = -68.0340;\n", + "Iter 48;\t Fid = 0.50;\t Obj = -38.8294;\n", + "Iter 49;\t Fid = 0.50;\t Obj = -80.6787;\n" + ] + } + ], + "source": [ + "cumulative_cost = 0.0\n", + "\n", + "with botorch.settings.validate_input_scaling(False):\n", + " for it in range(N_ITER):\n", + " mll, model = initialize_model(train_x, train_obj, m=1)\n", + " fit_gpytorch_mll(mll)\n", + " acqf = AugmentedUpperConfidenceBound(\n", + " model,\n", + " beta=3,\n", + " maximize=True,\n", + " best_f=train_obj[torch.where(train_x[:, -1] == 0)].min(),\n", + " cost={i: fid + 5.0 for i, fid in enumerate(fidelities)},\n", + " )\n", + " new_x = optimize_aucb(acqf)\n", + " if model.n_true_points < model.max_n_cheap_points:\n", + " new_x[:, -1] = fidelities.shape[0] - 1\n", + " train_x = torch.cat([train_x, new_x])\n", + "\n", + " new_x[:, -1] = fidelities[new_x[:, -1].int()]\n", + " new_obj = problem(new_x).unsqueeze(-1)\n", + " train_obj = torch.cat([train_obj, new_obj])\n", + "\n", + " print(\n", + " f\"Iter {it};\"\n", + " f\"\\t Fid = {new_x[0].tolist()[-1]:.2f};\"\n", + " f\"\\t Obj = {new_obj[0][0].tolist():.4f};\"\n", + " )" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:39:03.821762700Z", + "start_time": "2023-12-18T11:37:36.108340600Z" + } + }, + "id": "8d02e319798a28d7" + }, + { + "cell_type": "markdown", + "source": [ + "## Comparison to MES" + ], + "metadata": { + "collapsed": false + }, + "id": "ab5e775efb0ccfe9" + }, + { + "cell_type": "code", + "execution_count": 10, + "outputs": [], + "source": [ + "def initialize_mes_model(train_x, train_obj, data_fidelity):\n", + " model = SingleTaskMultiFidelityGP(\n", + " train_x,\n", + " train_obj,\n", + " outcome_transform=Standardize(m=1),\n", + " data_fidelity=data_fidelity,\n", + " )\n", + " mll = ExactMarginalLogLikelihood(model.likelihood, model)\n", + " return mll, model" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:39:03.827771200Z", + "start_time": "2023-12-18T11:39:03.821256200Z" + } + }, + "id": "1c93f20bdaffccac" + }, + { + "cell_type": "code", + "execution_count": 11, + "outputs": [], + "source": [ + "def optimize_mes_and_get_observation(mes_acq, fixed_features_list):\n", + " candidates, acq_value = optimize_acqf_mixed(\n", + " acq_function=mes_acq,\n", + " bounds=problem.bounds,\n", + " q=1,\n", + " num_restarts=5,\n", + " raw_samples=128,\n", + " fixed_features_list=fixed_features_list,\n", + " )\n", + " # observe new values\n", + " cost = cost_model(candidates).sum()\n", + " new_x = candidates.detach()\n", + " new_obj = problem(new_x).unsqueeze(-1)\n", + " return new_x, new_obj, cost" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:39:03.834820500Z", + "start_time": "2023-12-18T11:39:03.827771200Z" + } + }, + "id": "1a1bba8e3496b54d" + }, + { + "cell_type": "code", + "execution_count": 12, + "outputs": [], + "source": [ + "train_x_mes = torch.clone(train_x[:10])\n", + "train_x_mes[:, -1] = fidelities[train_x_mes[:, -1].int()]\n", + "train_obj_mes = torch.clone(train_obj[:10])" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:39:03.840874700Z", + "start_time": "2023-12-18T11:39:03.834820500Z" + } + }, + "id": "e8414dfbd0643afa" + }, + { + "cell_type": "code", + "execution_count": 13, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iter 0;\t Fid = 0.50;\t Obj = -57.5522;\n", + "Iter 1;\t Fid = 0.50;\t Obj = -45.4163;\n", + "Iter 2;\t Fid = 0.50;\t Obj = -9.0258;\n", + "Iter 3;\t Fid = 1.00;\t Obj = -48.1935;\n", + "Iter 4;\t Fid = 0.50;\t Obj = -0.9326;\n", + "Iter 5;\t Fid = 0.50;\t Obj = -80.6658;\n", + "Iter 6;\t Fid = 0.50;\t Obj = -2.1240;\n", + "Iter 7;\t Fid = 0.50;\t Obj = -10.3696;\n", + "Iter 8;\t Fid = 0.50;\t Obj = -4.4469;\n", + "Iter 9;\t Fid = 0.50;\t Obj = -8.9561;\n", + "Iter 10;\t Fid = 0.50;\t Obj = -22.0102;\n", + "Iter 11;\t Fid = 0.50;\t Obj = -16.9033;\n", + "Iter 12;\t Fid = 0.50;\t Obj = -13.5406;\n", + "Iter 13;\t Fid = 0.75;\t Obj = -0.6079;\n", + "Iter 14;\t Fid = 0.50;\t Obj = -31.6025;\n", + "Iter 15;\t Fid = 0.50;\t Obj = -38.2686;\n", + "Iter 16;\t Fid = 0.50;\t Obj = -17.9994;\n", + "Iter 17;\t Fid = 0.50;\t Obj = -19.7949;\n", + "Iter 18;\t Fid = 0.50;\t Obj = -275.5655;\n", + "Iter 19;\t Fid = 0.50;\t Obj = -111.8389;\n", + "Iter 20;\t Fid = 0.50;\t Obj = -91.2188;\n", + "Iter 21;\t Fid = 0.50;\t Obj = -76.4585;\n", + "Iter 22;\t Fid = 0.50;\t Obj = -0.5961;\n", + "Iter 23;\t Fid = 0.50;\t Obj = -14.7103;\n", + "Iter 24;\t Fid = 0.50;\t Obj = -2.7677;\n", + "Iter 25;\t Fid = 0.50;\t Obj = -53.0683;\n", + "Iter 26;\t Fid = 0.50;\t Obj = -1.3054;\n", + "Iter 27;\t Fid = 0.50;\t Obj = -23.7584;\n", + "Iter 28;\t Fid = 0.50;\t Obj = -12.2472;\n", + "Iter 29;\t Fid = 0.50;\t Obj = -12.6918;\n", + "Iter 30;\t Fid = 0.50;\t Obj = -4.2754;\n", + "Iter 31;\t Fid = 1.00;\t Obj = -1.7593;\n", + "Iter 32;\t Fid = 1.00;\t Obj = -44.1689;\n", + "Iter 33;\t Fid = 0.50;\t Obj = -108.9165;\n", + "Iter 34;\t Fid = 0.50;\t Obj = -143.6092;\n", + "Iter 35;\t Fid = 1.00;\t Obj = -3.5225;\n", + "Iter 36;\t Fid = 0.50;\t Obj = -4.6244;\n", + "Iter 37;\t Fid = 0.50;\t Obj = -2.1491;\n", + "Iter 38;\t Fid = 1.00;\t Obj = -189.5238;\n", + "Iter 39;\t Fid = 1.00;\t Obj = -0.9491;\n", + "Iter 40;\t Fid = 1.00;\t Obj = -33.1628;\n", + "Iter 41;\t Fid = 0.50;\t Obj = -86.3697;\n", + "Iter 42;\t Fid = 1.00;\t Obj = -5.4117;\n", + "Iter 43;\t Fid = 1.00;\t Obj = -3.7590;\n", + "Iter 44;\t Fid = 0.50;\t Obj = -12.7589;\n", + "Iter 45;\t Fid = 0.50;\t Obj = -4.9297;\n", + "Iter 46;\t Fid = 1.00;\t Obj = -17.4007;\n", + "Iter 47;\t Fid = 0.50;\t Obj = -46.4645;\n", + "Iter 48;\t Fid = 1.00;\t Obj = -16.0177;\n", + "Iter 49;\t Fid = 1.00;\t Obj = -2.7148;\n" + ] + } + ], + "source": [ + "candidate_set = torch.rand(\n", + " 1000, problem.bounds.size(1), device=problem.bounds.device, dtype=problem.bounds.dtype\n", + ")\n", + "candidate_set = problem.bounds[0] + (problem.bounds[1] - problem.bounds[0]) * candidate_set\n", + "\n", + "cumulative_cost = 0.0\n", + "\n", + "with botorch.settings.validate_input_scaling(False):\n", + " for it in range(N_ITER):\n", + " mll, model = initialize_mes_model(train_x_mes, train_obj_mes, data_fidelity=2)\n", + " fit_gpytorch_mll(mll)\n", + " acqf = qMultiFidelityMaxValueEntropy(\n", + " model, candidate_set, cost_aware_utility=cost_aware_utility\n", + " )\n", + " new_x, new_obj, cost = optimize_mes_and_get_observation(acqf,\n", + " fixed_features_list=[{2: fid} for fid in fidelities])\n", + " train_x_mes = torch.cat([train_x_mes, new_x])\n", + " train_obj_mes = torch.cat([train_obj_mes, new_obj])\n", + " cumulative_cost += cost\n", + " print(\n", + " f\"Iter {it};\"\n", + " f\"\\t Fid = {new_x[0].tolist()[-1]:.2f};\"\n", + " f\"\\t Obj = {new_obj[0][0].tolist():.4f};\"\n", + " )" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:41:01.885178100Z", + "start_time": "2023-12-18T11:39:03.842873900Z" + } + }, + "id": "fcaf20bf41f1b680" + }, + { + "cell_type": "markdown", + "source": [ + "## Plot results" + ], + "metadata": { + "collapsed": false + }, + "id": "dd44be2238fc0110" + }, + { + "cell_type": "code", + "execution_count": 14, + "outputs": [], + "source": [ + "mapping_fid = dict(zip(range(fidelities.shape[0]), fidelities.tolist()))\n", + "cost_AGP = torch.cumsum(torch.tensor([mapping_fid[int(source)] for source in train_x[:, -1].tolist()]), dim=0)\n", + "cost_MES = torch.cumsum(train_x_mes[:, -1], dim=0)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:41:01.892898500Z", + "start_time": "2023-12-18T11:41:01.887864Z" + } + }, + "id": "5c5cc1ef1808ad1f" + }, + { + "cell_type": "code", + "execution_count": 15, + "outputs": [], + "source": [ + "train_obj[torch.where(train_x[:, -1] != fidelities.shape[0] - 1)] = train_obj.min()\n", + "best_seen_AGP = torch.cummax(train_obj, dim=0)[0]" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:41:01.920142Z", + "start_time": "2023-12-18T11:41:01.893908500Z" + } + }, + "id": "4a7f7020f195a31f" + }, + { + "cell_type": "code", + "execution_count": 16, + "outputs": [], + "source": [ + "train_obj_mes[torch.where(train_x_mes[:, -1] != 1)[0]] = train_obj_mes.min()\n", + "best_seen_MES = torch.cummax(train_obj_mes, dim=0)[0]" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:41:01.921142200Z", + "start_time": "2023-12-18T11:41:01.907620600Z" + } + }, + "id": "f696548b9b3f50a5" + }, + { + "cell_type": "code", + "execution_count": 17, + "outputs": [ + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwsAAAJECAYAAABZ37i3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAB7CAAAewgFu0HU+AAB0K0lEQVR4nO3dd3hUddrG8XvSQyqhQ0KXJr2p9CYqLoK6KioiiIquqCgudtHdVVFWUVdRQAyuiiDqgoAgloCCKL1JUUqQQDCQUNKTSc77B29GpiQEksyck3w/15VrZ059BmeTuefXbIZhGAIAAAAAF36+LgAAAACAOREWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAgArWr18/2Ww22Ww2rVy50tflAECpERYAAOfl7A++nn78/PwUERGhJk2aaPjw4Zo1a5bS09N9XTYA4AIQFgAA5cowDGVkZCgxMVGLFi3S3XffrYsuukhffPGFr0sDAJynAF8XAACwrm7duql79+5O2woLC3Xy5Elt3bpVO3fulCT98ccfuu6667Rw4UL95S9/8UWpAIALQFgAAFywIUOG6Nlnny12/5o1azRixAglJSWpoKBA99xzjw4cOKDAwEDvFWkCjFMAYFV0QwIAVJiePXtqwYIFjueHDx/mgzMAWAhhAQBQoS699FI1adLE8byoaxIAwPwICwCAClevXj3H48zMTLf9iYmJjtmUGjdu7Ni+evVq3XnnnWrVqpWioqJks9k0YcIEp3MLCwv1ww8/6JlnntHgwYPVsGFDVatWTcHBwapXr54GDBig559/XsePHy9VrWfP7FRkz549mjBhglq3bq3w8HBFRkaqQ4cOevzxx0t13dJMnTp69GjHMXPmzJEkZWVlafr06erVq5fq1Kmj4OBgxcXF6eabb9aaNWtK9XoAoCwYswAAqHBHjx51PK5bt+45j8/Ly9MDDzygGTNmlHhcfn6+mjRposOHDxd736NHjyohIUEvvvii3nnnHY0cOfK8an/nnXc0YcIE5ebmOm3ftm2btm3bplmzZmn58uXq2rXreV33XHbu3Km//vWv2rVrl9P2pKQkzZs3T/PmzdMzzzyj5557rlzvCwBnIywAACrUhg0btH//fsfz3r17n/Ochx56yBEU2rVrpw4dOigwMFC//vqr/Pz+bBQvKChwBIXw8HBdfPHFatq0qSIjI5Wfn6+kpCT99NNPOn36tDIzM3XbbbcpMDBQN910U6lqnzNnju69915JUsuWLdW1a1eFhoZq9+7dWrNmjQzDUGpqqq655hrt2rVLUVFRpf53KcmRI0c0aNAgJScnKzo6Wr1791bdunV1/Phxfffddzp16pQk6R//+IfatGlT6tcDAOfNAADgPPTt29eQZEgyJk+eXOKx69atMxo3buw4/tprr/V43IEDBxzH+Pv7G5KMuLg44/vvv3c7Nicnx/E4NzfXGDNmjJGQkGDk5eV5vHZOTo7x8ssvGwEBAYYkIzo62khPTy+25qI6JBnBwcFGrVq1jGXLlrkdt2rVKiMyMtJx7HPPPVfsNc/+N0tISPB4zO233+50X0nGo48+amRmZjodl5qaagwYMMBxbNOmTY3CwsJi7w0AZUHLAgDggn355ZduffYLCwt16tQpbdu2TTt27HBsv/baa/Xhhx+e85oFBQWqVq2avvnmG7Vo0cJtf3BwsONxUFCQ3nvvvRKvFxwcrL///e8qLCzUY489ppMnT+qDDz5wtBicyzfffKP27du7be/Tp49eeOEFjR8/XpL08ccf65lnninVNc8lNzdXjz/+uF544QW3fTExMZo7d66aNWumzMxM7d+/X+vWrdMll1xSLvcGgLMRFgAAF2z9+vVav359icfUq1dP06dP1/Dhw0t93fHjx3sMCmUxZswYPfbYY5LOBIDShIW7777bY1AoMmrUKE2YMEF2u1179uzR6dOnFRkZWeZaa9WqVWLwqFOnjq6++mp98sknkkRYAFBhCAsAgAqVnJys66+/XrfccoveeOMNVa9e/ZznjBgx4rzvU1hYqI0bN2rLli1KSkrS6dOnlZ+f7/HYLVu2lOqaN9xwQ4n7IyIi1KxZM+3Zs0eGYejgwYNq167d+ZbuZujQoQoJCSnxmE6dOjnCQmJiYpnvCQCeEBYAABds8uTJHldwzszMVGJiopYtW6aXX35Zx44d04cffqjNmzfrhx9+KDEwBAYGntcHbrvdrjfeeEPTpk1TUlJSqc4p7TSqpamjRo0ajsenT58u1XXNel8AcMU6CwCAchcWFqaLL75YjzzyiDZv3qwGDRpIkn755Rc9/PDDJZ5bvXp1BQSU7rus3NxcXX311Zo4cWKpg4Ikpaenl+q40sxuFBgY6HhcXEvG+fLVfQHAFWEBAFChGjRooMmTJzuef/jhh07rLrgKDQ0t9bWfe+45rVixQtKZxdRuuukmffLJJ9q1a5dOnTqlvLw8GYbh+Cly9uOSnL0wmzf56r4A4IpuSACACnfFFVc4Htvtdq1atarMawPk5ubqP//5j+P5nDlzNGrUqGKPL21rAgDgT7QsAAAqXL169ZyeHzx4sMzXXLdunTIyMiRJF198cYlBobzuCQBVDWEBAFDhsrKynJ6fvQrzhTpy5IjjcWkGBH///fdlvicAVDWEBQBAhdu0aZPT86IBz2VxduBwDSOuCgsLNXPmzDLfEwCqGsICAKDCTZs2zfHYZrNpwIABZb5m06ZNHY9XrVqlU6dOFXvs1KlTtXXr1jLfEwCqGsICAKDCnDx5UuPGjdPixYsd22655RbVqVOnzNfu1KmTo4Xi1KlTuuGGG5y6JklnBkE/88wzeuyxxxQWFlbmewJAVcNsSACAC/bll196XOAsKytLiYmJ+umnn5Sdne3Y3qJFC7366qvlcm8/Pz/985//1B133CFJ+vrrr9WiRQv16NFDjRo1UmpqqlauXKkTJ05IkmbOnKlbb721XO4NAFUFYQEAcMHWr1+v9evXl+rYa665RjNmzFDt2rXL7f5jxozR3r179cILL0g6s3L0119/7XRMSEiIXnvtNd1yyy2EBQA4T4QFAEC5Cw4OVlRUlJo3b65LL71Ut9xyi7p06VIh93r++ed11VVX6c0339Tq1at17NgxRUREKDY2VldeeaXGjh2riy66qELuDQCVnc0o7TKWAAAAAKoUBjgDAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8KCRRw8eFATJ05Uq1atFBYWppiYGHXr1k1Tp05VVlaWr8sDAABAJWQzDMPwdREo2eLFizVy5EidPn3a4/4WLVpo6dKlat68uZcrAwAAQGVGWDC5zZs3q2fPnsrOzlZ4eLgef/xx9e/fX9nZ2Zo3b55mzZol6Uxg2LBhgyIiInxcMQAAACoLwoLJ9enTRz/88IMCAgL0/fff67LLLnPaP3XqVE2aNEmSNHnyZD377LM+qBIAAACVEWHBxNatW6dLLrlEkjRu3Di98847bscUFhaqbdu22rVrl6Kjo5WSkqLAwMByuX9OTo62b98uSapVq5YCAgLK5boAAAAof3a7XceOHZMktWvXTiEhIWW+Jp/+TGzhwoWOx2PGjPF4jJ+fn0aNGqXHH39cJ0+eVEJCggYPHlwu99++fbu6d+9eLtcCAACA96xbt07dunUr83WYDcnEVq9eLUkKCwtTly5dij2ub9++jsdr1qyp8LoAAABQNdCyYGK7du2SJDVv3rzELkCtWrVyO6c81KpVy/F43bp1qlevXrldGwAAAOUrOTnZ0Svk7M9xZUFYMKmcnBwdP35ckhQbG1visdWrV1dYWJgyMzN16NChUt8jKSmpxP1Ffd4kqV69euesAwAAAOZQXmNNCQsmlZ6e7ngcHh5+zuOLwkJGRkap7xEXF3dBtQEAAKBqYMyCSeXk5DgeBwUFnfP44OBgSVJ2dnaF1QQAAICqhZYFkzp7qqu8vLxzHp+bmytJCg0NLfU9ztVl6ex+bwAAAKh6CAsmdfZKzKXpWpSZmSmpdF2WijAGAQAAACWhG5JJhYSEqEaNGpLOPRD5xIkTjrDAOAQAAACUF8KCibVp00aStHfvXtnt9mKP2717t+Nx69atK7wuAAAAVA2EBRPr1auXpDNdjDZu3FjscatWrXI87tmzZ4XXBQAAgKqBsGBiw4cPdzyOj4/3eExhYaH++9//SpKio6PVv39/b5QGAACAKoCwYGLdu3dX7969JUmzZ8/W2rVr3Y555ZVXHKs2P/jggwoMDPRqjQAAAKi8mA3J5F5//XX17NlT2dnZGjx4sJ544gn1799f2dnZmjdvnmbOnClJatGihSZOnOjjagEAAFCZEBZMrlOnTpo/f75Gjhyp06dP64knnnA7pkWLFlq6dKnTdKsAAABAWdENyQKGDh2qbdu26aGHHlKLFi1UrVo1RUdHq2vXrnrppZe0efNmNW/e3NdlAgAAoJKxGYZh+LoImFNSUpJj3YZDhw6xiBsAAICJVcRnN7ohAQAswzAMZecXKD3HrvSc/P//X7sycj0/P/O/duUXFPq6dABVXIs6EfrHsLa+LuO8ERYAAKXy/a/HNPWrPUpJz/H6vQ1DyrUXKiPXroJCGsQBWE+e3ZpfWhAWAADntHjrEU2Yv4UP6gBQxRAWAAAlWrDhkB79bJvICQBQ9RAWAADF+uCng3p64Q5fl1GskEA/hQcHKjIkQOEhAYoICVB4cIAiQgIVERKgiP9/HBTA5H8AfKtWRLCvS7gghAUAgEfv/rBf/1q6y237yEsbanCbul6vJyjA7/8DwJkgEBYcQAgAgApGWAAAuPnPt7/pla9/ddt+T99mevTKlrLZbD6oCgDgbYQFAICDYRia+tUeTV+5z23fw5e30P0DmhMUAKAKISwAACSdCQr/WLJT8WsS3fY9MaSV7u7TzPtFAQB8irAAAFBhoaEnF+7Qx+t+d9v3j2EXa9Rljb1fFADA5wgLAFDF2QsKNenTbfp882Gn7Tab9NJ17XVjtzgfVQYA8DXCAgBUYXn2Qk2Yv1lfbj/qtN3fz6ZXb+ygYR0b+KgyAIAZEBYAoIrKyS/Q+Lmb9M2uFKftgf42/efmzrqyrfenRwUAmAthAQCqoOy8At39wQb98Ntxp+3BAX5657Yu6t+yto8qAwCYCWEBAKqYjFy77ohfr3WJaU7bqwX5691RXdWjeU0fVQYAMBvCAgBUIaey8nV7/DptOXTSaXtEcIDix3RT18YxvikMAGBKhAUAqEImLtjiFhSiQgP1wdjuah8b7ZOa4COGIe1eIiVvPfMYlVuHm6WazX1dBSyIsAAAVcTB1Ey3wcw1w4P0wdhL1LpepI+qgs9890/ph1d8XQW8peFlhAVcEMICAFQRGxJPOD2PCAnQvLsvU/Pa4T6qCD7zxy/S6mm+rgKABfj5ugAAgHds/N05LPRsVpOgUBUZhrT8Mcko9HUlACyAlgUAqCI2urQsdGlU3UeVwKd2L5UOfO+8reFlUmR939QD74io4+sKYFGEBQCoAk5l5+vXlHSnbZ0JC1VPfo604knnbZGx0sjPpaBqvqkJgKnRDQkAqoAth046TXgTFOCntg0Y1Fzl/DRdOpHovG3wPwgKAIpFWACAKmDjQecuSO0bRCk4wN9H1cAnTidL3//beVvDy6SLr/NNPQAsgbAAAFXApoOMV6jyvv2HlJ951gabdOUUyWbzWUkAzI+wAACVXEGhoc0uMyExXqGKSdoobZ3rvK3TSKl+R5+UA8A6CAsAUMntPnpamXkFTts6NyQsVBmFhdKySc7bgiKkgc/4ph4AlkJYAIBKzrULUqMa1VQrIthH1cDrti+QDm9w3tZ3khRe2zf1ALAUwgIAVHKug5sZr1CF5GZI30x23hbTTLrkHt/UA8ByCAsAUMm5rtxMWKhCVk+T0pOdt13xghQQ5Jt6AFgOYQEAKrGU0zk6lJbttI2wUEWcSJR+/I/ztmYDpRZX+KQcANZEWACASmyTS6tCRHCALqod4aNq4FUrnpYKcv98bvOXrnyRqVIBnBfCAgBUYhsSncNCx4bR8vfjw2Kld+B7adcXztu63y3VaumbegBYFmEBACoxxitUQQV2afnjzttCY6R+j/qmHgCWRlgAgEoqJ79AOw6fctrWtVGMj6qB12x6X/pjh/O2AU9JoQRFAOePsAAAldSOw6eUX2A4nvvZpA5xUT6sCBUu+4T03b+ct9VpK3UZ7ZNyAFgfYQEAKinX9RVa1o1UREigj6qBV6x8ScpOc9525YuSn79v6gFgeYQFAKik3Bdji/ZNIfCOY3uk9bOct7UeKjXp45t6AFQKhAUAqIQMw2Dl5qrEMM4Mai60/7nNP1i6/J++qwlApUBYAIBK6GBqllIz85y2Mbi5EvtthbTvW+dtPcZLMU18Uw+ASoOwAACVkGurQq2IYMVWD/VRNahQ9jz3qVLD60q9HvZNPQAqFcICAFRCbusrNKwuGyv3Vk7rZkhp+5y3Xf6cFBzum3oAVCoBvi4AxUtMTNTixYu1cuVKbdu2TYcPH1ZhYaFq1qyprl27asSIEfrrX/+qgAD+MwJwtsnK4xUKC6Xtn0jJ2yQZ5zy8ytv8ofPzBl2ldjf6phYAlQ6fMk3q6aef1vPPPy/DcP9DefjwYR0+fFiLFi3Sq6++qk8//VQNGzb0QZUAzOhUdr72/JHutK2zlcLCyhek76f6ugrruuolyY+OAwDKB79NTCo5OVmGYSgsLEwjR45UfHy8Vq9erQ0bNuiDDz5Qt27dJEnr16/XoEGDlJGR4eOKAZjFlkMndfb3DEEBfmrbINJ3BZ2vXxb6ugLr6nCzFNvV11UAqEQICyZVo0YNvfTSS0pOTtYHH3yg0aNHq2fPnurSpYtGjhyptWvX6sYbzzQz//bbb3r11Vd9XDEAs3Ad3Ny+QZSCAyy0KFf2iXMfA3fVakoDJ/u6CgCVDN2QTOqll14qcb+/v7+mT5+uhQsXKi8vT59++qmeeeYZL1UHwMwsPV5BkvIynZ+3+otUjWlfS1StptTxVimynq8rAVDJEBYsrEaNGmrfvr02bNigffv2nfsEAJVeQaGhzS4zIVlqvEJhgWTPdt424Gmpdivf1AMAVRzdkCwuNzdX0pmWBgDYczRdmXkFTts6N7RQWMjPct8WVM37dQAAJBEWLC0lJUW7du2SJLVu3drH1QAwg40H05yeN6pRTbUign1UzQXI8xQWWC8AAHyFbkgWNnXqVNntdklyDHY+H0lJSSXuT05OvqC6APiO6+DmLlZqVZCkPA8zuwXSsgAAvkJYsKiff/5Zr732miQpNjZW995773lfIy4urpyrAuBrbis3N7ZaWHAZ3GzzlwIs1DICAJUM3ZAs6I8//tBf//pX2e122Ww2vf/++6pWjW/egKou5XSODqU5Dw623ExIrmMWgsIlm803tQAAaFkoK1s5/BGLj4/X6NGjS3Vsenq6rr76akcXoilTpmjAgAEXdN9Dhw6VuD85OVndu3e/oGsD8L5NLq0KEcEBuqh2hI+quUCu3ZAY3AwAPkVYsJCcnBwNGzZMGzdulCQ98sgjmjRp0gVfLzY2trxKA2ACruMVOjaMlr+fxb6Vdx3gHBTmmzoAAJIIC2VWNBtRWdSrd+5FdOx2u2688UYlJCRIku68805NnTq1zPcGUHlssPpibJL7mAUGNwOATxEWyqhVq4pfKKiwsFC33XabFi9eLEm66aabNGPGjAq/LwDryMkv0I7Dp5y2WTMsuHZDYtpUAPAlBjhbwLhx4zRv3jxJ0tChQ/Xhhx/Kz4//dAD+tOPwKeUXGI7nfjapY1y07wq6UG4DnOmGBAC+xCdOk3v44Yf17rvvSpIGDhyoBQsWKCCABiEAzlzHK7SsG6mIkEAfVVMGrt2QGOAMAD5FWDCxZ599VtOmTZMk9ejRQ4sWLVJwMPONA3Dnthhbo2jfFFJWbmGBbkgA4Et8RW1S//nPf/Tcc89Jkho0aKCXX35ZBw4cKPGcli1bKjDQgt8kAigTwzDcpk215HgFiQHOAGAyhAWT+uyzzxyPDx8+rF69ep3znAMHDqhx48YVWBUAMzqYmqXjGXlO27o0jPFRNWXk1rLAmAUA8CW6IQGAxbl2QaoZHqy4mFAfVVNGnlZwBgD4DC0LJrVy5UpflwDAIja6dEHq2qh6uawu7xOs4AwApkLLAgBY3KbKsBhbEVZwBgBTISwAgIWdzsnXnj/SnbZ1tnRYcB3gTFgAAF8iLACAhW3+/aSMP9diU5C/n9o2iPRdQWXl1g2JsAAAvkRYAAALcx3c3C42SsEB/j6qphy4DXBmzAIA+BJhAQAsrFKNV5BYlA0ATIawAAAWVVBoaHNlWYxNkgoLPbQs0A0JAHyJsAAAFrXnaLoy8wqctnVuaOGw4BoUJFZwBgAfIywAgEW5rq/QqEY11YoI9lE15cC1C5JENyQA8DHCAgBY1MbENKfnXazcqiBJ+Z7CAi0LAOBLhAUAsCjXlgVLr68gubcs2PykgBDf1AIAkERYAABLSjmdo0Np2U7bLD24WfKwenO4ZLP5phYAgCTCAgBY0iaXVoWI4AC1qBPho2rKieuCbAxuBgCfIywAgAW5LsbWsWG0/P0s/i0806YCgOkQFgDAglzDguW7IEkeFmSjZQEAfI2wAAAWk5NfoB2HTzttqxxhwaUbEtOmAoDPERYAwGJ2HD6lvIJCx3ObTeoYF+27gsqL2wBnuiEBgK8RFgDAYly7ILWsE6GIkEAfVVOOXLshMcAZAHyOsAAAFuMaFro2rgRdkCT3RdnohgQAPkdYAAALMQzDbdrUSjFeQWKAMwCYEGEBACzkUFq2jmfkOW3r0jDGR9WUM7ewwJgFAPA1wgIAWMjWpJNOz2PCghQXE+qbYsqb25gFwgIA+BphAQAsZJtLWGgfGyWbzeKLsRWhZQEATIewAAAWsjXplNPz9rHRvimkIrCCMwCYDmEBACyioNDQjsPOYaFDbJSPqqkAtCwAgOkQFgDAIvYdy1BWXoHTtkrVsuC2gjNhAQB8jbAAABax9dBJp+cNokNVKyLYN8VUBNcVnFmUDQB8jrAAABaxzW28QiXqgiR56IbEomwA4GuEBQCwCPeZkKJ9UkeFKCxkgDMAmBBhAQAsINdeoJ3Jp522VarBzfZsSYbzNlZwBgCfIywAgAXsTk5XfoHzh+m2lSksuHZBkuiGBAAmQFgAAAtw7YLUtFaYIkMCfVNMRfAUFhjgDAA+R1gAAAtwXYytQ2UaryB5CAs2KTDUJ6UAAP5EWAAAC3Af3FyJuiBJHgY3h0s2m29qAQA4EBYAwOQyc+3am+K8YFmlmglJ8rAgG12QAMAMCAsAYHI7Dp9S4VljmwP8bLq4fqTvCqoIbmssMG0qAJgBYQEATM51MbYWdSIUEujvo2oqiNvqzYQFADADwgIAmNxWl/EKHeIq2XgFyUM3JMICAJgBYQEATM61ZaHSjVeQPAxwZswCAJgBYQEATOxEZp5+T3P+IF3pZkKSGLMAACZFWAAAE9t22LlVITjATy3qRPiomgrkFhZYvRkAzICwAAAmtu3QSafnF9ePVKB/JfzV7RoWWL0ZAEyhEv7FAYDKw3Xl5ko5XkGiGxIAmBRhwYKWLVsmm83m+Hn22Wd9XRKACuK6cnOlnAlJkvIJCwBgRoQFi8nMzNS9997r6zIAeMHRUzlKSc912kbLAgDAmwgLFvP000/r4MGDql27tq9LAVDBXNdXiAgOUJMalfRDtOuibIQFADAFwoKFbNy4UW+88YaCg4P1/PPP+7ocABXMtQtSu9go+fnZfFNMRXMb4ExYAAAzICxYREFBge666y4VFBToiSeeUPPmzX1dEoAKViUWYyvCCs4AYEqEBYuYNm2aNm/erBYtWujRRx/1dTkAKphhGG5hoUNlXIytCCs4A4ApERYsIDExUZMnT5Ykvf322woODvZxRQAq2sHULJ3Kznfa1j4u2jfFeAOLsgGAKREWLODee+9VVlaWbr31Vg0YMMDX5QDwAtfBzTXDg1Q/KsQ3xVQ0w2A2JAAwqQBfF4CSzZ07V8uXL1d0dLReffXVcr12UlJSifuTk5PL9X4ASs/TeAWbrZIObs7PlmQ4b2MFZwAwBcKCiaWlpemhhx6SJL344ovlPl1qXFxcuV4PQPlxnQmpfWUer+DaqiDRDQkATIJuSCb2yCOPKCUlRZdcconuvvtuX5cDwEvsBYXacfi007YOlXkmJNfVmyUGOAOASdCyUEbl0S0gPj5eo0ePdtq2cuVKxcfHy9/fX++88478/Mo/1x06dKjE/cnJyerevXu53xdAyX5LyVB2foHTtqrVsmCTAkJ9UgoAwBlhwYRyc3M1btw4SdIDDzygjh07Vsh9YmNjK+S6AMrGtQtSg+hQ1QivxLOgeVq9uQK+IAEAnD/CQhnt2rWrzNeoV6+e0/PPP/9cv/76qwIDA9WmTRvNmzfP7ZydO3c6Hu/YscNxzCWXXKImTZqUuSYAvrPVdX2FuErcqiC5L8jG4GYAMA3CQhm1atWq3K+Zm5srScrPz9ddd911zuM/++wzffbZZ5LOdGkiLADW5j64OdondXgN06YCgGnRzgsAJpKTX6DdyelO2yr1eAXJw+rNhAUAMAvCggmNHj1ahmGU+JOQkOA4fvLkyY7trgOlAVjLruTTshf+ueaAzSa1a1DJw4JrNyTCAgCYhs+6IZ0+fVrp6ekqKCg457ENGzb0QkUA4Huui7E1rRmmiJBAH1XjJa4DnBmzAACm4dWw8PXXX2v69OlavXq10tLSSnWOzWaT3W6v4MoAwBy2uoxXqNTrKxRhzAIAmJbXwsIDDzygt956S5JkGMY5jgaAqsm1ZaHSj1eQPHRDYvVmADALr4SFuXPn6s0335QkhYSEaPjw4erSpYtiYmIqZLExALCijFy79h1z/uDcPi7aN8V4k9sAZ7ohAYBZeCUszJgxQ5IUFxen7777Ts2aNfPGbSu1fv360UIDVDLbk07p7P9bB/jZ1KZepO8K8ha6IQGAaXnla/1t27bJZrNp8uTJBAUAKIbr+got60YoJNDfN8V4k2tYCCQsAIBZeCUs5OfnS5I6derkjdsBgCW5j1eI9k0h3kbLAgCYllfCQuPGjSVJGRkZJR8IAFWY60xIHeOqwOBmiUXZAMDEvBIWrrvuOknSt99+643bAYDlpGbkKulEttO2qtOywKJsAGBWXgkLEydOVMOGDfXaa69p9+7d3rglAFjKtsPOXZBCAv10Ue0qMoUo3ZAAwLS8EhaioqL01VdfqU6dOurRo4emT5+uEydOeOPWAGAJ2w45h4W29aMU4F9FppZmBWcAMC2vTJ3atGlTSVJWVpZOnjyp+++/Xw888IBq1qypatVK/qNgs9m0b98+b5QJAD7jOhNSlemCJHloWagiLSoAYAFeCQuJiYlOzw3DkGEYSklJOee5NputgqoCAHMwDENbXWZC6lBVBjcbhpRPNyQAMCuvhIXbb7/dG7cBAEtKPpWj4xm5TtuqTMuCPUcyCp23sYIzAJiGV8JCfHy8N24DAJbk2gUpMiRAjWtUkQ/Mrl2QJLohAYCJVJHRcwBgXq5dkNrHRledLpiewgIDnAHANAgLAOBj7oObq8h4BYmwAAAm55VuSK6ys7O1ceNGHT16VFlZWRo+fLgiIyN9UQoA+FRhoaFtHloWqgzX1ZsDq0l+fI8FAGbh1bBw6NAhPfHEE1qwYIHy8/Md27t27ao2bdo4ns+ePVszZsxQVFSUVqxYUXWa4wFUOYmpmUrPsTttqzIzIUms3gwAJue1r29+/vlnderUSXPnzlVeXp5j+lRPhg4dqm3btum7777TihUrvFUiAHida6tCrYhg1Y0M8VE1PsDqzQBgal4JCydPntSwYcOUlpamunXravr06dq+fXuxx9euXVtXXXWVJGnp0qXeKBEAfGLLoZNOzzvERlWt1lS31ZsJCwBgJl7phvTGG28oJSVFNWvW1Nq1a9WwYcNznjNo0CAtWrRI69at80KFAOAbVXrlZoluSABgcl5pWVi8eLFsNpsefvjhUgUFSbr44oslSfv27avI0gDAZ/ILCvXLkdNO26rUTEiS+wBnFmQDAFPxSljYu3evJKlPnz6lPqd69eqSpNOnT5/jSACwpl//SFeu3Xn14qrXsuA6ZoEF2QDATLwSFnJyciRJgYGBpT4nM/PMH5DQ0NAKqQkAfM11cHNcTKhiwoJ8VI2P0A0JAEzNK2Ghdu3akqQDBw6U+pwtW7ZIkurXr18RJQGAz1X58QqShwHOdEMCADPxSli45JJLJEnLli0r1fGGYWjWrFmy2Wzq3bt3RZYGAD6z9ZBzy0KHqjZeQWLqVAAwOa+EhVtvvVWGYeijjz5ytBiUZOLEidq6dask6fbbb6/g6gDA+3LyC7Tnj3SnbVWyZSGfsAAAZuaVsDBs2DD1799fdrtdAwcO1Ntvv62UlBTHfrvdriNHjmjBggXq3bu3Xn/9ddlsNl133XXq0aOHN0oEAK/65chpFRT+uTClzSa1bUDLAmEBAMzFK+ssSNJnn32mgQMHavPmzRo/frzGjx/vWHioU6dOTscahqFLL71Uc+bM8VZ5AOBVruMVmtcKV3iw134lmwdhAQBMzSstC5IUHR2ttWvX6vHHH1dkZKQMw/D4ExoaqkmTJmnlypUKC+OPBoDKyXUmpCrZBUlyDwus4AwApuLVr7GCgoL0/PPP64knntCqVau0YcMGpaSkqKCgQDVq1FCnTp00aNAgRUVVwaZ4AFVGTn6BNh484bStQ1wV/b1HywIAmJpP2rzDwsI0ZMgQDRkyxBe3BwCfWbP3uJ7833b9nuY8ZWiVbVlgBWcAMLUq2EEWALwvLTNP/1q6U59vOuy2LyI4QK3rRfigKhNgBWcAMDWfhIXs7Gxt3LhRR48eVVZWloYPH67IyEhflAIAFcowDH2+6bD+tXSnTmTlu+0P8LPpmaFtFBzg74PqfMww6IYEACbn1bBw6NAhPfHEE1qwYIHy8//8o9m1a1e1adPG8Xz27NmaMWOGoqKitGLFCsesSQBgJYnHM/Xkwu1aszfV4/7ODaP14nXt1bJuFW1VsOdKRoHzNlZwBgBT8VpY+Pnnn3X11VfrxIkTMoyz5xZ3DwJDhw7Vfffdp/z8fK1YsUJXXHGFt8oEgDLLLyjUzO/3641vf1OuvdBtf0RwgCZd1Uq3dm8oP78q/GWIa6uCRDckADAZr0ydevLkSQ0bNkxpaWmqW7eupk+fru3btxd7fO3atXXVVVdJkpYuXeqNEgGgXGw8eEJ/eWO1pn61x2NQuKptXX0zsa9uu7RR1Q4KkvvqzRIDnAHAZLzSsvDGG28oJSVFNWvW1Nq1a9WwYcNznjNo0CAtWrRI69at80KFAFA2p3PyNXX5Hn3480Gd1XjqUC8qRP8Y1laXt6nj/eLMylPLAt2QAMBUvBIWFi9eLJvNpocffrhUQUGSLr74YknSvn37KrI0ACiz5TuOavIXO/TH6Vy3fTabNLpHY00c3LJqrtBckjyXaVMDQiW/KjjQGwBMzCt/ufbu3StJ6tOnT6nPqV69uiTp9OnTFVITgKojYU+KVu5OUV6Be7egsjqUlq3Ve4973Ne6XqSmXNdOHeKiy/2+lUJehvNzZkICANPxSljIycmRJAUGBpb6nMzMM83ToaGhFVITgKohYXeKxsxZ79V7hgT66aFBLXRHryYK9PfK0DBrYtpUADA9r/wVq127tiTpwIEDpT5ny5YtkqT69etXREkAqoivfjnq1fv1aVFLXz/UV+P6NiMonIvb6s2EBQAwG6/8JbvkkkskScuWLSvV8YZhaNasWbLZbOrdu3dFlgagkjue4T6OoCLUCAvS6yM66v0x3RQXwyDdUqEbEgCYnle6Id1666369NNP9dFHH+nBBx9Ux44dSzx+4sSJ2rp1q2w2m26//XZvlAigknJdNbln8xpqUad8F0FrWitcQ9vXU3S1oHK9bqXnOsCZmZAAwHS8EhaGDRum/v37KyEhQQMHDtS//vUvXX/99Y79drtdR44c0Zo1a/TGG2/oxx9/lM1m03XXXacePXp4o0QAldSJzDyn57d0b6Sr29fzUTVw4jZmgQXZAMBsvDaP32effaaBAwdq8+bNGj9+vMaPH+9YvblTp05OxxqGoUsvvVRz5szxVnkAKqkTWc5hoXq10k+0gApGNyQAMD2vjb6Ljo7W2rVr9fjjjysyMlKGYXj8CQ0N1aRJk7Ry5UqFhfGHo0hmZqbeeustDRw4UA0aNFBwcLDq1Kmjzp076/7779eKFSt8XSJgOgWFhk5mO3dDoquQibgNcKYbEgCYjVdXCAoKCtLzzz+vJ554QqtWrdKGDRuUkpKigoIC1ahRQ506ddKgQYMUFRXlzbJMLyEhQWPGjNHBgwedtqekpCglJUWbN2/WDz/8oMGDB/uoQsCcTmfnu62mHBNGWDANuiEBgOn5ZDnRsLAwDRkyREOGDPHF7S3lm2++0dChQ5WTk6Po6Gjdc8896tevn2rXrq2srCzt2rVLS5Ys0R9//OHrUgHTSXPpgiRJ0XRDMg/XsMAAZwAwHZ+EBZTOsWPHNGLECOXk5Khjx45avny56tSp43RMz549deeddyovz/1DEVDVnXQJC9WC/BUS6O+jauCGRdkAwPRMExb++OMPLVmyRMePH1eTJk30l7/8RdWqVe1vmR5//HGlpqaqWrVqWrhwoVtQOFtQEF0rAFdpmc7jFaozXsFcCAsAYHpeCQu7du3S5MmTZbPZNGPGDEVHRzvt/+KLL3TLLbcoOzvbsS02NlaLFi0655oMldWJEyc0d+5cSdLIkSPVqFEjH1cEWI/bTEhhdEEylXzCAgCYnVdmQ1q4cKE+/fRTHTlyxC0opKSkaOTIkcrKynKaFenQoUMaOnSoMjIyPF+0kluyZIkjPF1zzTWO7VlZWdq7d6+OHj0qw3XkJgAnrmss0LJgMrQsAIDpeSUsfPvtt7LZbPrLX/7itm/69OnKyMhQQECAXn31VW3dulUvv/yy/Pz8dOTIEc2aNcsbJZrOTz/95Hjcrl07rV+/XoMHD1ZERIQuuugi1atXT3Xq1NH48eMZ3AwUw3X1ZsKCybit4ExYAACz8Uo3pN9//12S++Jr0pnF2mw2m0aNGqUJEyZIOvPh+LffftOsWbP0xRdf6KGHHvJGmaayc+dOx+OEhATdeeedstvtTsccO3ZMb731lj777DMtX75cHTp0OK97JCUllbg/OTn5vK4HmI17ywLdkEyFlgUAMD2vtCykpKRIkmrXru20/fjx4/rll18kSbfccovTvqKuN2d/aK5K0tLSHI/vuece2Ww2/etf/9Lvv/+u3Nxc/fLLLxo9erQk6ejRoxo+fLhOnz59XveIi4sr8ad79+7l+ZIAr3Mfs0DLgmkYhocVnKv2pBYAYEZeCQtFfe9zcnKctq9evVrSmZl8evXq5bSvXr16kqSTJ09WfIEmlJn55zduOTk5mj17tp588knFxcUpKChIbdq0UXx8vO6++25JUmJiot5++21flQuYkltYoBuSeRTkSUaB8zYWZQMA0/FKWIiJiZH0Z3ekIt9++60kqWvXrm5TfxZ1uQkPN/cfD5vNVuafOXPmuF03JCTE8bh9+/a67bbbPN7/hRdeUHBwsCRp/vz551X7oUOHSvxZt27deV0PMBu3MQu0LJiHaxckiW5IAGBCXhmz0KFDB3399deaO3eubrzxRklnWhsWLFggm82mAQMGuJ1z8OBBSSpxbYHKLCIiwvF48ODBxR5Xo0YNde3aVWvWrNHWrVuVl5dX6jUXYmNjy1wnYGaMWTAxT2GBFZwBwHS8EhZGjBihFStWaPHixRoxYoR69eql+fPnKyUlRX5+frr55pvdzvn5558lyfTrC+zatavM1yjqcnW2uLg4x4xIcXFxJZ5ftL+wsFBpaWmqW7dumWsCrK6w0NDJbGZDMi1aFgDAErwSFkaNGqX33ntPq1ev1oIFC7RgwQLHvjFjxqhVq1Zu53z++eey2Wzq0aOHN0q8YJ5qLw8XX3yx49+poKCgxGPP3h8QYJpFuQGfSs+xq6DQeS0SuiGZiGtYCAiR/Px9UwsAoFheGbPg5+enZcuW6eGHH1ZsbKwCAgIUFxenp59+2uOg3CVLligxMVGSNGTIEG+UaDp9+vRxPN6/f3+Jx+7bt0/SmXEOReNDgKrOdXCzJMXQsmAerN4MAJbgta+hw8LC9O9//1v//ve/z3lsz549deDAAUnm74ZUUfr06aNatWrp2LFjWrx4sV577TX5+7t/63bgwAFt2bJF0pl/Nz8/r+Q/wPTSXMJCcICfQoP45to0WGMBACzBlJ8sq1evrkaNGlXZoCBJ/v7+euSRRySdGez9z3/+0+0Yu92uv/3tbyosLJR0Zj0GAGecdAkLMXRBMhfXsMDqzQBgSqYMCzjjgQceUOfOnSVJzz33nG6++WYtX75cmzZt0oIFC9SnTx8tX75c0pnuWtdff70vywVMJS3TeXBzNF2QzIWWBQCwBEbDmlhISIiWLFmioUOHauPGjZo3b57mzZvndtyQIUM0b9482Ww2H1QJmJN7ywLTpppKfpbzc1ZvBgBTomXB5OrVq6effvpJ77zzjvr27atatWopMDBQdevW1TXXXKPPP/9cS5cudVqXAYD7AGdaFkwmL8P5Oas3A4Ap0bJgAQEBARo3bpzGjRvn61IAy3DthsRMSCZDNyQAsARaFgBUSq7dkFi92WTyXLohsXozAJgSYQFApZSW6RIWmA3JXNxaFuiGBABmRFgAUCmdzHLuhlSdbkjm4rYoGy0LAGBGhAUAlZLromy0LJgMYxYAwBIICwAqHcMwGLNgdnRDAgBL8EpYaNKkiZo1a6a9e/eW+pzff/9dTZs2VbNmzSqwMgCVUUauXfkFhtM2uiGZjNsKznRDAgAz8srUqQcPHpTNZlNeXt65D/5/+fn5SkxMZKExAOfNdbyCRDck06EbEgBYAt2QAFQ6rjMhBfn7KSzI30fVwCO3FZwJCwBgRqYNC6dOnZIkVatG0zSA8+O+enMgrZRm47aCM2EBAMzItGHhww8/lCQ1atTIx5UAsBrXsBBDFyTzYcwCAFhChYxZGDBggMftY8aMUVhYyd8e5ebmav/+/UpJSZHNZtPgwYMrokQAldiJTOcxC9HMhGQu9jyp0O68jdmQAMCUKiQsrFy5UjabTYbx52wkhmFo/fr153Wdpk2b6vHHHy/v8gBUcrQsmJxrFySJbkgAYFIVEhb69Onj1D941apVstls6tKlS4ktCzabTSEhIapXr5569OihESNGnLMlAgBcuY9ZICyYiuvgZokVnAHApCqsZeFsfn5nhkbMmTNHbdq0qYhbAoCDazekGMKCubiOV5CkQL4YAgAz8so6C6NGjZLNZlP16tW9cTsAVZyn2ZBgIq7dkPyDJX+v/DkCAJwnr/x2njNnjjduAwCS3NdZYMyCyeSxxgIAWIWpvsrZt2+fjh8/rsaNG6tOnTq+LgeARbmu4Fydbkjm4rZ6MzMhAYBZeWWdhZSUFE2fPl3Tp093LLZ2tr1796pLly5q0aKFevTooQYNGuj666/XiRMnvFEegErEMAyluXRDqk7Lgrnku4YFBjcDgFl5JSx8/vnnGj9+vF5//XVFRUU57cvNzdVVV12lLVu2yDAMGYahwsJCLVy4UMOGDfNGeQAqkez8AuXZC522VWfMgrm4tSzQDQkAzMorYWHFihWy2Wy69tpr3fbNmTNH+/btkyRdc801ev311zV06FAZhqE1a9Zo/vz53igRQCXhOl5BomXBdFi9GQAswythYc+ePZKkSy+91G3f3LlzJZ1Z9XnhwoW6//77tWjRIg0aNEiGYWjevHneKBFAJeE6XiHAz6aIYFMNzwJjFgDAMrwSFo4dOyZJio2NddqenZ2tn376STabTXfffbfTvjvuuEOStGnTJm+UCKCScG1ZiK4W5LRIJEyAbkgAYBleCQsnT548czM/59v99NNPys/Pl81m06BBg5z2NWnSRNKZwdEAUFquaywwXsGEXFdwZoAzAJiWV8JCePiZJuajR486bS9a6blNmzZuC7YFBp75Ax8QQPcBAKV3IpOZkEzPdVE2uiEBgGl5JSy0atVKkrR8+XKn7Z999plsNpv69u3rdk5RsGC9BQDn44TbGgu0LJiO66JsDHAGANPyytf2V199tX766SfNnDlTrVu3Vu/evTVnzhzt3LlTNptN1113nds5RWMVGjRo4I0SAVQSrt2QWL3ZhBizAACW4ZWwMH78eE2fPl3JyckaP368077LLrtM/fv3dztn8eLFstls6tatmzdKBFBJuLYsRLN6s/kQFgDAMrzSDSkqKkrffPONOnfu7Fh4zTAM9e7dW5988onb8Vu3btX69eslSZdffrk3SgRQSbiNWaAbkvm4reBMWAAAs/La6OHWrVtrw4YNOnDggI4ePap69eqpcePGxR4fHx8v6cz6CwBQWu6zIdGyYDq0LACAZXh9qqEmTZo4pkUtTocOHdShQwcvVQSgMnFvWSAsmI7bAGfCAgCYlVe6IQGAt7jNhsQAZ/NxmzqVsAAAZuX1loXCwkIlJCRo7dq1Onr0qLKysvT888+rXr16jmPy8vJkt9vl7++v4OBgb5cIwKJy8guUnV/gtI0xCybk1g2JqVMBwKy8GhaWLFmiBx54QAcPHnTa/sgjjziFhXfffVf333+/wsPDdeTIEYWF8a0TgHNzHa8gMXWq6djzpELn1h8WZQMA8/JaN6RZs2Zp2LBhSkxMlGEYqlGjhgzD8HjsnXfeqaioKGVkZOh///uft0oEYHFpLuMV/GxSZAgtC6biOhOSRDckADAxr4SF3377Tffdd5+kM7Mb7dy5UykpKcUeHxQUpOuvv16GYWjFihXeKBFAJXDSwxoLfn42H1UDj1wHN0us4AwAJuaVsDBt2jTZ7XZdfPHF+vLLL9WqVatzntO7d29J0ubNmyu6PACVhGvLQjTjFczHdbyCRMsCAJiYV8LCd999J5vNpgkTJigoqHT9h5s3by5JOnToUEWWBqASOekyZiGGaVPNx3UmJP8gyZ9QBwBm5ZWwkJSUJEnntXZC0aDmrCwPTdYA4EFapns3JJhMvsvvdFoVAMDUvBIWbLYzfYbP54N/amqqJCkqKqpCagJQ+bjOhhQTxjfWpuM2bSozIQGAmXklLDRo0ECStH///lKfs3r1aklS06ZNK6QmAJWPa1hg9WYTcg0LDG4GAFPzSljo16+fDMPQ+++/X6rjT506pXfeeUc2m00DBgyo4OoAVBas3mwBbi0LdEMCADPzSlgYN26cbDabVq1apTlz5pR4bGpqqoYPH66jR48qICBA99xzjzdKBFAJnMh0bVmgG5LpEBYAwFK8EhY6deqkBx98UIZhaOzYsbrpppv0ySefOPb/+OOPmjt3ru677z41b95c33//vWw2m55++mk1atTIGyWa2vLlyzVixAg1bdpU1apVU0hIiOLi4jRs2DDNnz9fhYWFvi4RMAW6IVmA66JshAUAMLUAb93olVdeUW5urt5++219+umn+vTTTx0Dn8eNG+c4rmhV5wkTJuipp57yVnmmlJubq1tvvVWfffaZ276kpCQlJSXpiy++0FtvvaUvvvhC0dHR3i8SMBG3lgW6IZkPLQsAYCleaVmQzsyI9NZbb+mrr75Sv379ZLPZZBiG048kXXbZZVq6dKleffVVb5VmWg888IAjKNSuXVv//ve/9d133+mHH37Q9OnTHa0uP/zwg0aMGOHLUgGfy7MXKjOvwGkbLQsm5LqCMwOcAcDUvNayUOTyyy/X5ZdfrvT0dG3evFkpKSkqKChQjRo11LFjR9WsWdPbJZnSH3/8oXfffVeSVL16dW3cuFGxsbGO/b169dKtt96qDh06KDExUV999ZU2bNigrl27+qpkwKdcF2STGLNgSq6LsjF1KgCYmtfDQpGIiAj16dPHV7c3vZ9//tkxFmHMmDFOQaFIZGSkHnroIT344IOSpLVr1xIWUGWluYQFm02KCiUsmI5bNyRaFgDAzLzWDQnnJy/vzw8+Ja010axZM4/nAFXNCZfVmyNDAhXgz68402EFZwCwFNP8JT1x4oSOHTvmGLtQ1bVs2dLxuKTF7Pbt2+fxHKCqcV+9mfEKpuS2KBthAQDMrELDgt1u144dO7Rx40YdO3bMbX9OTo6eeeYZxcbGqmbNmqpbt64iIiL017/+Vb/88ktFlmZ67dq1U48ePSRJc+bM0ZEjR9yOSU9P12uvvSbpTOvD4MGDvVkiYCquYSGa8QrmxGxIAGApFTJmwTAMTZ48WW+88YbS09Md2y+77DJNmzZN3bp1U15enq644gqtXr3acY4kZWVl6X//+5+WLVumL774QgMHDqyIEi0hPj5eV155pQ4cOKDOnTtr0qRJ6ty5swICArRjxw69/PLLOnDggGrWrKmPPvpIQUHn901qUlJSifuTk5PLUj7gVa7TpsYwE5I5ERYAwFIqJCyMGTNGH3zwgSQ5dSv68ccfdeWVV+rnn3/W9OnT9cMPP0iSYmJidNFFF8lut2vnzp3Kzs5Wdna2br31Vu3Zs0dRUVEVUabptWjRQuvXr9fbb7+tl156SRMnTnTaHxgYqEceeUQPPvigxwHQ5xIXF1depQI+dyLLecxCNGHBnBizAACWUu7dkBISEvTf//5XkhQcHKzrr79ejzzyiG644QaFhobq5MmTmjZtmubMmaPAwEDNnDlTx44d09q1a7V+/XodP35cjzzyiCTp2LFjmjNnTnmXaCmLFy/WRx99pIyMDLd9+fn5+uSTTzR37lzGeqDKc2tZCKMbkim5TZ1KWAAAMyv3sBAfHy/pzCJiGzdu1IIFC/Tyyy9r/vz52rhxo+rUqaOZM2fq1KlTeuihh3TnnXc6VnKWpNDQUL388su64oorZBiGli5dWt4lliubzVbmn+IC0cSJEzVmzBjt3r1bw4cP15o1a5SRkaHs7Gxt2rRJY8aM0e+//65HH31Uf/3rX1VQUODxOsU5dOhQiT/r1q0rh38hwDvcxyzQsmBKbgOcmToVAMys3MPCzz//LJvNpoceekitW7d22teqVSs99NBDjg+1t912W7HXuf322yWpyg50PnsV69GjR+t///ufevToobCwMIWEhKhTp05677339PTTT0uSPv/8c02fPv287hEbG1viT7169cr9dQEVJc2lGxKzIZlQQb5U4DLFM4uyAYCplfuYhaJZey677DKP+8/e3rx582Kvc9FFF0mS0tLSyrG68rdr164yX8PTh/Ki1ZttNpv+9a9/FXvuE088oWnTpikjI0Pvvfee7r///jLXA1iR6wrOrN5sQq6tChLdkADA5Mo9LGRmZspmsykmJsbj/ujoaMfj4ODgYq8TEhIiyfwLjbVq1apCrlsUQmrXrq0GDRoUe1xISIguvvhi/fzzz9q9e3eF1AJYQVqma1igZcF0XAc3S6zgDAAmV2HrLJw9DqE02+EsIOBMjrPb7ec8Nj8/3+kcoKrJLyhUeo7z/1eq0w3JfDy1LLAoGwCYmmlWcIazJk2aSJJSU1NL7OqUlpamHTt2OJ0DVDUnXcYrSLQsmJLrTEh+gVIA/50AwMwICyY1dOhQx+MJEyZ47I5VWFioBx54wLHvL3/5i9fqA8zEdbyCxArOppTHGgsAYDUV1m9l+vTpql27ttv2lJQUx+N//OMfxZ5/9nFV0ejRo/Xaa69p165dWrFihbp27ar7779fHTp0kL+/v3bu3Km3335ba9eulSTVqVNHDz/8sI+rBnzDdbxCREiAAv35LsR03FZvZiYkADC7CgsLb7/9drH7isYtPPfccxV1e8sLCgrSsmXLNGzYMG3dulXbt2/X3Xff7fHYJk2a6PPPP1fNmjW9XCVgDq6rN9MFyaTyXcMCg5sBwOwqJCywmnD5aNSokdavX6958+bp008/1aZNm3Ts2DEZhqGYmBi1b99ew4cP16hRoxQWRnM+qi7XBdkY3GxSbi0L/N4CALMr97CQkJBQ3pes0gIDA3XbbbeVuIAdUNW5hQXGK5iT2+rNhAUAMLtyDwt9+/Yt70sCQIlOuIxZiKEbkjnRsgAAlsMIQACW5zpmIZqwYE5uYYExCwBgdoQFAJbn1rIQRjckU3JdwZmWBQAwPcICAMtzHbNAy4JJuS7KxtSpAGB6hAUAlufaDSmG2ZDMyW2AM92QAMDsCAsALM+9ZYFuSKbECs4AYDmEBQCWVlBo6FQ2LQuW4NYNibAAAGZHWABgaaey8+W6DiQrOJsUA5wBwHIICwAsLc1lJiSJbkimxToLAGA5hAUAlnbSZbxCWJC/ggP8fVQNSsQKzgBgOYQFAJbm2rLAtKkmRssCAFgOYQGApZ1k2lTrYAVnALAcwgIAS0tj2lRrKLBLBbnO21iUDQBMj7AAwNJc11igZcGk8jPdt9ENCQBMj7AAwNJOuIxZYNpUk3JdkE1iBWcAsADCAgBLO+EyZoGwYFKu4xUkuiEBgAUQFgBYmlvLQhhjFkzJdfVmvwApgGAHAGZHWABgaa5jFmhZMClWbwYASyIsALA0uiFZBAuyAYAlERYAWFZhoeG2gjPdkEyKBdkAwJIICwAs63ROvgoN5220LJgUYQEALImwAMCyXLsgSYQF0yIsAIAlERYAWFaay0xIIYF+Cg3y91E1KJHromyEBQCwBMICAMtyHa8QQ6uCebkNcGZBNgCwAsICAMtybVmIJiyYl+sKzizIBgCWQFgAYFknXcYsxIQRFkzLdVE2uiEBgCUQFgBYluuCbNHVmDbVtNwGONMNCQCsgLAAwLJcwwItCybGCs4AYEmEBQCWdSLTuRsSYxZMzLUbEis4A4AlEBYAWFaa22xIdEMyLbcBzoQFALACwgIAy3KdOrU63ZDMi0XZAMCSCAsALCvNpRsSqzebGGEBACyJsADAkgzDcG9ZICyYFys4A4AlERYAWFJ6rl32QsNpW/UwxiyYltsKzoQFALACwgIASzrp0gVJomXBtAoLJHuO8zZaFgDAEggLACzJdSakoAA/VQvy91E1KJFrq4JEWAAAiyAsALAk1wXZqlcLlM1m81E1KBFhAQAsi7AAwJJOZDK42TJcV2+WCAsAYBGEBQCWdCKLaVMtw3X1Zpu/5M9/LwCwAsICAEtybVmIYUE283JbvTlcossYAFgCYQGAJbmOWYiuxrSppuW2IFs139QBADhvhAUAluQaFmhZMDEWZAMAyyIsALCkEy7rLEQzZsG83FoWCAsAYBWEhQqQkZGh77//Xv/+97914403qkmTJrLZbLLZbGrcuPF5X2/Hjh0aN26cmjVrptDQUNWqVUu9e/fWO++8I7vdXv4vALAA95YFuiGZFqs3A4BlBfi6gMpo6NChWrlyZblca9asWRo/frzy8v78YJSTk6PVq1dr9erVio+P19KlS1WzZs1yuR9gFe5jFmhZMC1aFgDAsmhZqACGYTgex8TEaPDgwQoPDz/v63z55Ze65557lJeXpzp16uiNN97Qzz//rGXLlum6666TJK1bt07XXnutCgoKyq1+wOwMw3DrhhRDWDAvBjgDgGXRslABbrnlFo0bN07dunVT8+bNJUmNGzdWRkbGOc78U35+vu6//34VFhYqMjJSa9asUbNmzRz7r7zySt13332aPn26Vq9erQ8++ECjR48u75cCmFJWXoHyCgqdtrHOgom5LsoWdP5fngAAfIOWhQpw99136+abb3YEhQvxv//9T/v375ckPf74405BocjUqVNVvXp1x2OgqkhzWWNBkqozZsG8XBdloxsSAFgGYcGkFi5c6HhcXItBtWrVdOONN0qSdu7cqV9//dULlQG+d9Jl9eYAP5vCg2koNS23Ac50QwIAqyAsmNTq1aslSS1btlTdunWLPa5v376Ox2vWrKnwugAzSHMZ3Fw9LEg2VgQ2L08rOAMALIGwYEIZGRk6dOiQJKlVq1YlHnv2/l27dlVoXYBZnHQNC6zebG5u3ZBoWQAAq6Dd3oSSkpIcj2NjY0s8Ni4uzvG4KGBcyH08SU5OPq/rAd7iOmaBwc0m5zbAmTELAGAVhAUTSk9Pdzw+15SrYWF//tE9n9mWJOegAVjJCZcxC4QFk3ObOpVuSABgFXRDMqGcnBzH46Cgkj8EBQcHOx5nZ2dXWE2AmZxwbVkIIyyYmms3JAY4A4BlVNmWhfIYDBkfH18haxuEhIQ4Hp+9crMnubm5jsehoaHndZ9zdVtKTk5W9+7dz+uagDe4rt7MmAWTcxvgTDckALCKKhsWzCwiIsLx+FxdizIz/2zeP99Vos81HgIwK9ewEEPLgrm5dUMiLACAVVTZsFAeMwfVq1evHCpx16BBA8fjcw1CPrt1gDEIqCpOZDqPWYhmzIJ5FRZIdpcukoQFALCMKhsWzjUlqS9FREQoLi5Ohw4d0u7du0s89uz9rVu3rujSAFNwb1mgG5Jpuc6EJDFmAQAshAHOJtWrVy9J0p49e3T06NFij1u1apXjcc+ePSu8LsAMXMMCLQsm5toFSWI2JACwEMKCSQ0fPtzxeM6cOR6PycrK0ieffCJJatOmjVq0aOGFygDfys4rUE5+odM2pk41MY9hgW5IAGAVhAWTuvbaa9W0aVNJ0osvvqh9+/a5HfP3v/9dJ06ccDwGqgLXVgVJiiEsmJdrWLD5SQHBno8FAJhOlR2zUJH27t2r1atXO20rmtUoIyPDraXgyiuvVN26dZ22BQYG6j//+Y+GDh2q06dPq2fPnnrqqafUvXt3nThxQrNmzdJnn30m6UyXpdtuu63iXhBgIq6rN/vZpIgQfpWZltvqzeFSOUxdDQDwDv7CVoDVq1drzJgxHvelpqa67UtISHALC5I0ZMgQvfPOOxo/frz++OMP3X///W7HdO/eXf/73//k7+9fPsUDJnfSw+rNfn58+DQtFmQDAEujG5LJ3XXXXdq4caPuuusuNW3aVCEhIapRo4Z69eqlt99+W2vWrFHNmjV9XSbgNWlug5uZCcnUWGMBACyNloUKMHr06HJd2blt27aaOXNmuV0PsLKTLMhmLazeDJhSdna2Tp8+rczMTBUUFPi6HJTA399fYWFhioyMVGhoqNfvT1gAYCmuYxaYNtXkXLshERYAnzt16pSOHDni6zJQSna7Xbm5uUpLS1P9+vUVFRXl1fsTFgBYiuuYBWZCMjm3Ac6EBcCXsrOz3YJCQAAfB83Mbrc7Hh85ckTBwcEKCQnx2v15dwCwFLeWBVZvNjfXMQsMcAZ86vTp047HkZGRqlu3LpOkmFxBQYGOHj3q+G936tQpr4YFBjgDsBTXdRZoWTA5twHOrN4M+FJm5p//nyQoWIO/v7/TrJln/zf0BsICAEtxDQus3mxyzIYEmErRYOaAgACCgoX4+/s7uot5e0A6YQGApZzIdFlngdmQzM0tLNANCQCshLAAwFLcWxYYs2BqnlZwBgBYBmEBgGXk2guUlefc/ErLgsmxgjMAWBphAYBluE6bKjFmwfRYlA0ALI2wAMAyXKdNtdmkqFC6IZkasyEBMLE77rhDNptNNWrUUG5ubonHbtmyRffcc4/atGmjyMhIBQUFqW7durr88sv1yiuv6NixY27n2Gw2p5+AgADVq1dPw4cP1/fff19RL6tcsc4CAMtwHa8QFRoofz+bj6pBqbit4Ew3JADmkJ6erk8++UQ2m01paWlauHChbrrpJrfjCgsLNWnSJL3yyivy9/dXnz59NHjwYIWFhSklJUVr167VI488osmTJ2vPnj1q0KCB0/k1atTQ+PHjJUk5OTnasmWLFi1apC+++ELz58/XDTfc4JXXe6EICwAsw3UmJNZYsABWcAZgUvPnz1dmZqYefvhhvfbaa5o9e7bHsPDkk0/qlVdeUefOnTV//nw1b97c7ZhNmzbp0UcfVXZ2ttu+mjVr6tlnn3Xa9u677+quu+7SpEmTTB8W6IYEwDJcWxaimQnJ/NxWcCYsADCH2bNnKyAgQJMmTVL//v317bff6uDBg07H/Prrr5o6dapq1aql5cuXewwKktS5c2d9/fXXaty4canufccddygsLEyJiYkeuy+ZCS0LACzjhMuYhRhmQjK3wkJaFgCLKCw03L6QMbPq1YLkV4ZuqDt37tRPP/2kIUOGqE6dOho1apS+/fZbxcfHO7UCvP/++yooKNC4ceNUq1atc163aOG082Gzmbs7LWEBgGWccJkNKZpuSObmGhQkxiwAJnUiK09d/vWNr8sotY1PDVKN8OALPn/27NmSpNtuu02SdN111+lvf/ub4uPj9cwzz8jP70znm7Vr10qS+vfvX8aKnb3//vvKzMxUkyZNVLNmzXK9dnkjLACwDNdvvWhZMDnXLkgSsyEB8Ln8/Hx98MEHioyM1PDhwyVJ4eHhuvbaa/Xhhx/qm2++0eDBgyVJR48elSTVr1/f7TorV67UypUrnbb169dP/fr1c9p2/PhxR2tFTk6Otm7dquXLl8vPz09Tp04t19dWEQgLACyDMQsWk+8pLNANCYBvLVq0SMeOHdPYsWMVEhLi2D5q1Ch9+OGHmj17tiMslGTlypV67rnn3La7hoXU1FTHcf7+/qpZs6aGDRumiRMnqnfv3mV7MV7AAGcAluE2ZoFuSObm1rJgkwJCPB4KAN5S1AVp1KhRTtsHDhyoBg0aaNGiRUpLS5Mk1alTR5J05MgRt+s8++yzMgxDhmHo448/LvZ+LVu2dBxnt9t19OhRLVy40BJBQaJlAYCFMGbBYtxWbw4/s5IeANOpXi1IG58a5OsySq36Bf7+P3TokFasWCFJ6tu3b7HHffjhh3rggQfUo0cPrVy5UgkJCRowYMAF3dPqCAsALIPZkCyGBdkAy/Dzs5VpwLBVzJkzR4WFherVq5datmzptt9ut+v999/X7Nmz9cADD+j222/XlClTNHPmTD344IOmH4xcEQgLACwhv6BQ6bl2p23VGbNgbq7dkBivAMCHDMNQfHy8bDab3n//fTVt2tTjcb/++qvWrl2rDRs2qGvXrpo0aZKmTJmiq666Sh9//LHHtRZOnjxZwdX7DmEBgCV4mv+7Oi0L5sYaCwBM5LvvvtOBAwfUt2/fYoOCJI0ZM0Zr167V7Nmz1bVrVz3//PPKy8vTq6++qlatWqlPnz7q0KGDqlWrppSUFG3btk3r1q1TeHi4Onbs6L0X5CUMcAZgCSddxitIUnQoLQum5toNidWbAfhQ0cDm0aNHl3jcTTfdpNDQUH388cfKzs6Wn5+fXnnlFW3atEljx45VcnKy3n33XU2dOlWLFy9WeHi4pk6dqn379jmmYq1MaFkAYAlpLuMVIkMCFODP9x2m5jbAmbAAwHfmzp2ruXPnnvO4yMhIZWW5LyrZqVMnzZgx47zuaRjGeR1vRvylBWAJJ126IdEFyQLcxiwwwBkArIawAMAS0jKduyFd6LR58CK32ZBYvRkArIawAMASXAc4MxOSBTDAGQAsj7AAwBJc11igG5IFuHZDCqQbEgBYDWEBgCW4rt5MNyQLcBuzQDckALAawgIAS3DthsTqzRbAAGcAsDzCAgBLcA0L0YxZMD/GLACA5REWAFiC65iFGLohmZ/bmAXCAgBYDWEBgCW4jlmIJiyYn9vUqYQFALAawgIA07MXFOpUtnNYYMyCBbCCMwBYHmEBgOm5BgWJdRYswW2AM2EBAKyGsADA9FwHN0t0QzK9wkIGOANAJUBYAGB6ruMVwoMDFBTAry9Ts2dLMpy3sSgbAFgOf20BmF6a2+rNdEEyPdcuSBKLsgGABREWAJjeSZduSKzebAEewwLdkAD4XmJiomw2m2w2m+rWrSu73e7xuF27djmOa9y4sWP7nDlzHNuL+xk9erTTtTIzM/XCCy+oc+fOCg8PV3BwsGJjY9W7d289/vjj2rdvXwW+4rIJ8HUBQHn5Zucf+v63Y8ovMM59MCxl99HTTs8JCxbgFhZsUmCoT0oBAE8CAgL0xx9/6Msvv9Q111zjtn/27Nny8yv+e/WBAweqV69eHvd17NjR8Tg9PV29evXStm3b1Lx5c40cOVI1atTQ8ePHtW7dOk2ZMkXNmjVTs2bNyvyaKgJhAZZXWGjopeW7NeP7/b4uBV7CTEgW4Glws83mm1oAwIMePXpo69ateu+999zCgt1u14cffqhBgwZp1apVHs8fNGiQHnvssXPe57XXXtO2bdt05513aubMmbK5/C48cOCAcnNzL/yFVDC6IcHScu0FmjB/C0GhiqkZHuzrEnAurguyMbgZgMmEhoZqxIgRWrp0qVJSUpz2LVmyRH/88YfuuOOOMt9n7dq1kqT77rvPLShIUpMmTdSqVasy36ei0LJQATIyMrRp0yatW7dO69at0/r165WYmChJatSokeNxSQoLC7V69WotX75cP/74o3bv3q20tDSFhISoYcOG6tOnj+655x61b9++Yl+MiZ3Kztc9H2zU2v2pvi4FXuTvZ9OVbev6ugycC2ssANZSWChlp/m6itILjZFK6CJUWnfccYdmzJihDz74QBMnTnRsf++99xQTE6Phw4eX+R41atSQJP36669O3ZOsgrBQAYYOHaqVK1eW6RqNGzfWoUOH3Lbn5+frl19+0S+//KIZM2bokUce0ZQpUzwm1cos+VS2Rr+3Xnv+SHfaHuBn0w1dYxUc4O+jylCRQgL9Nah1bXVtHOPrUnAubqs3MxMSYGrZadJUc/aZ9+jv+6SwmmW+TPfu3dW2bVvFx8c7wsLRo0e1bNky3XvvvQoOLr4l+5tvvlFOTo7HfSNGjHC0Ftxwww368MMPdeedd2rdunUaPHiwunTp4ggRZkdYqACG8ecA25iYGHXt2lU//vijMjIySjjL2ZEjRyRJzZs31/XXX6+ePXuqfv36ys7OVkJCgqZNm6YTJ07o5Zdflr+/v1544YVyfx1mtedoukbHr1PyKef/g4YF+Wv6yC7q26KWjyoD4ODaDSmIbkgAzOmOO+7Qww8/rJ9//lmXXHKJ3n//fdnt9nN2Qfr222/17bffetzXsWNHR1i45ppr9Morr2jy5Ml65ZVX9Morr0iSmjVrpiuvvFIPPvigLrroovJ9UeWIMQsV4JZbbtHcuXP122+/KTU1VV999dV5p8fu3btr+fLl+vXXXzVlyhQNHTpUXbp0Ua9evfT0009r/fr1qlXrzIfiqVOnav/+qtFn/8d9x/XXd350Cwq1IoI1f9xlBAXALFi9GYBFjBw5UoGBgXrvvfckSfHx8erUqdM5uwy9+OKLMgzD449r96WHH35YR44c0SeffKIJEyaoV69e+v333/XWW2+pffv2+uKLLyro1ZUdYaEC3H333br55pvVvHnzC77Gjz/+qCuuuKLY7kXNmjXTM888I+nMiP2FCxde8L2s4outRzT6vfVKz3GeD7lprTB9fm8PtW0Q5aPKALhxHbPAAGcAJlWrVi0NHTpU8+bN0zfffKM9e/aUy8BmVxEREbrhhhs0bdo0/fDDDzp27Jj+9re/KScnR2PHjlVeXt65L+IDdEOysP79+zsem3kxj7IyDEOzftivF77c7bavS6PqendUV1UPY959wFTcuiExZgEwtdCYM+MArCK0fMeujR07Vp9//rlGjx6tkJAQ3XrrreV6fU+ioqL05ptvaunSpTp48KC2b9+uLl26VPh9zxdhwcLOnpPX379yDugtKDT0zyU7NefHRLd9V1xcR6+P6KSQwMr52gFLcxvgTDckwNT8/MplwLBVXXHFFWrQoIEOHz6sESNGqHr16l65r81mU1iYuX8/EhYs7OxFQlq3bu3DSipGTn6BHpq/Rct2HHXbd/tljfTM0Ivl71e1ZoECLMNt6lS6IQEwL39/fy1cuFBJSUnlPr3pjBkz1LlzZ3Xr1s1t38KFC7Vr1y5FR0erbdu25Xrf8kJYsKisrCy99tprkqTg4GANGzbsvK+RlJRU4v7k5OQLKa1cnMzK053vb9CGgyfc9j12VSuN69O0yk0XC1hKvmtYoBsSAHPr2rWrunbtWurjS5o6tW7durrnnnskScuWLdM999yj5s2bO2a3zMzM1ObNm/XDDz/Iz89P06dPL3GaVl8iLFjUo48+qt9//13SmRUB69evf97XiIuLK++yysWhtCyNjl+nfcecP2wE+tv07xs6aFjHBj6qDECpMcAZQCVX0tSpHTp0cISFl156ST179tTXX3+t77//3vFlbIMGDXT77bfr/vvvN+VYhSI24+xFAVBhGjdurIMHD5Z6BeeSfPTRRxo5cqSkM92PNm7cqNDQ0PO+zvl8M3/o0CHFxsae9z3OS/I2Hdy/W//57jedzs532hUa6K97+zVTq7qRFVsDgPLx7T+k43v+fH71q1K3sb6rB4Ak6bfffpPdbldAQICp5/aHu9L8t0tKSnJ8GVxen92qbMtCeXRhiY+P1+jRo8tezHlYuXKlxo498wc3JiZGn3322QUFBUkeV4g+W3Jysrp3735B174QR76drkZ7P9a/JcnT5EarPGwDYA0McAYAS6qyYcGKNmzYoGuuuUa5ubkKDw/Xl19+WaaBzRXeUnAetiWd1I49KbqFiY2AyomwAACWVGXDwq5du8p8jXr16pVDJaXzyy+/6Morr1R6erqCg4O1cOFCXXLJJV67f0VrWz9K6TWqSSd9XQmA8meT6nf2dREAgAtQZcNCq1atfF1Cqe3bt0+XX365UlNTFRAQoPnz52vgwIG+Lqtc+fnZdEnbFkpeW185+QUKCw5QzfBgMTMqYHHVakiX3SdFMTEBAFhRlQ0LVpGUlKRBgwYpOTlZfn5+ev/99y9omlQrCBj0jMJ6Pa4l6w5pbK8m8iMpAAAA+BRhwcRSUlI0aNAgx+xJ77zzjm655RbfFlXBIkMCdVefpr4uAwAAAJL8fF0APDt58qSuuOIK7dlzZurBadOm6a677vJxVQAAAKhKaFmoAHv37tXq1audtmVkZDj+d86cOU77rrzyStWtW9fxPDc3V1dffbW2bNkiSbr11ls1aNAg7dixo9h7hoWFqUmTJuXzAgAAAAARFirE6tWrNWbMGI/7UlNT3fYlJCQ4hYXk5GT9+OOPjucfffSRPvrooxLv2bdvX61cufLCiwYAAJWev7+/7Ha77Ha7CgoK5O/PnOVWUFBQILvdLkle/29GNyQAAIAqIizszzVPjh49qoKCAh9Wg9IoKCjQ0aNHHc/P/m/oDbQsVIDRo0eXaWXnxo0byzCM8isIAABAUmRkpNLS0iRJp0+f1unTpxUQwMdBMytqUSgSFRXl1fvz7gAAAKgiQkNDVb9+fR05csSxzfXDKMyrfv36CgkJ8eo9CQsAAABVSFRUlIKDg3Xq1CllZmbSFcnk/P39FRYWpqioKK8HBYmwAAAAUOWEhIT45IMnrIcBzgAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjxjgjGKdPZVacnKyDysBAADAuZz9ea28psQlLKBYx44dczzu3r27DysBAADA+Th27JgaN25c5uvQDQkAAACARzbDMAxfFwFzysnJ0fbt2yVJtWrVYjn4/5ecnOxoaVm3bp3q1avn44pQGfC+QkXgfYWKwPvKvOx2u6NnSLt27cplLQ0+/aFYISEh6tatm6/LMLV69eopNjbW12WgkuF9hYrA+woVgfeV+ZRH16Oz0Q0JAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4xKJsAAAAADyiZQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERZQpaWkpGjJkiV65plndNVVV6lmzZqy2Wyy2WwaPXr0eV9v2bJluvbaaxUbG6vg4GDFxsbq2muv1bJly8q/eJjWhg0b9I9//EODBw92vBfCw8PVokULjRkzRqtXrz6v6/G+wunTpzVv3jxNnDhRffv2VfPmzRUVFaWgoCDVrl1b/fr108svv6zU1NRSXe/HH3/UyJEj1ahRI4WEhKhu3bq64oor9PHHH1fwK4FVPProo46/hzabTStXrjznOfyuqqQMoAqTVOzP7bffXurrFBQUGGPHji3xenfeeadRUFBQcS8GptC7d+8S3wdFP6NGjTJyc3NLvBbvKxT5+uuvS/W+qlmzprF8+fISrzV58mTDz8+v2GtcffXVRnZ2tpdeGcxo8+bNRkBAgNP7IiEhodjj+V1VudGyAPy/hg0bavDgwRd07pNPPqnZs2dLkjp16qSPP/5Y69at08cff6xOnTpJkt5991099dRT5VYvzOnIkSOSpPr16+vBBx/Up59+qnXr1mnt2rV69dVX1aBBA0nSf//733O2XvG+wtni4uI0atQovf766/r888+1du1arVmzRvPnz9cNN9wgf39/HT9+XNdcc422bt3q8RozZszQc889p8LCQjVr1kyzZ8/WunXrtHDhQvXv31+StHTpUt1xxx3efGkwkcLCQt19992y2+2qXbt2qc7hd1Ul5+u0AvjSM888YyxevNg4evSoYRiGceDAgfNuWdizZ4/jG5iuXbsaWVlZTvszMzONrl27GpKMgIAA47fffivvlwETufrqq4358+cbdrvd4/5jx44ZLVq0cLzPVq1a5fE43lc4W3Hvp7P973//c7yvrr32Wrf9qampRlRUlCHJaNiwoXHs2DG3ewwdOrRU3ySj8po2bZohyWjVqpXx+OOPn/P9wO+qyo+wAJzlQsLCvffe6zhn7dq1Ho9Zu3at45i//e1v5VgxrGjx4sWO98P999/v8RjeV7gQLVu2dHRHcvXSSy853i8ff/yxx/MPHTpk+Pv7G5KMIUOGVHS5MJmDBw8a4eHhhiRj5cqVxuTJk88ZFvhdVfnRDQkoA8MwtGjRIklSq1atdOmll3o87tJLL1XLli0lSYsWLZJhGF6rEeZT1N1Dkvbt2+e2n/cVLlRERIQkKScnx23fwoULJUmRkZG67rrrPJ4fGxurQYMGSZK+/fZbpaenV0yhMKX77rtPGRkZuv3229W3b99zHs/vqqqBsACUwYEDBxx91M/1i7Vo/+HDh5WYmFjRpcHEcnNzHY/9/f3d9vO+woXYs2ePtmzZIunMB7ez5eXlad26dZKkyy67TEFBQcVep+g9lZubqw0bNlRMsTCdTz75REuWLFFMTIz+/e9/l+ocfldVDYQFoAx27tzpeOz6x9nV2ft37dpVYTXB/FatWuV43Lp1a7f9vK9QWllZWfrtt9/06quvqm/fvrLb7ZKkCRMmOB3366+/qqCgQBLvKbg7efKkHnzwQUnSSy+9pJo1a5bqPH5XVQ0Bvi4AsLKkpCTH49jY2BKPjYuLczw+dOhQhdUEcyssLNSUKVMcz2+88Ua3Y3hfoSRz5szRmDFjit3/2GOP6ZZbbnHaxnsKJZk0aZKOHj2qnj17auzYsaU+j/dV1UBYAMrg7P684eHhJR4bFhbmeJyRkVFhNcHcpk2b5ugOct1116lLly5ux/C+woXo2LGjZs6cqW7durnt4z2F4vzwww969913FRAQoHfeeUc2m63U5/K+qhrohgSUwdmDCEvqAyxJwcHBjsfZ2dkVVhPMa9WqVXrsscckSbVr19bbb7/t8TjeVyjJ8OHDtX37dm3fvt0xl/21116rLVu26Oabb9aSJUvczuE9BU/y8vJ09913yzAMPfTQQ2rbtu15nc/7qmogLABlEBIS4nicl5dX4rFnD2oNDQ2tsJpgTr/88ouuvfZa2e12hYSEaMGCBcUueMT7CiWJjo5W27Zt1bZtW3Xr1k0jRozQ559/rv/+97/av3+/hg0bpjlz5jidw3sKnrzwwgvavXu3GjZsqMmTJ5/3+byvqgbCAlAGRdMUSuduVs3MzHQ8PldzLSqXAwcOaPDgwTpx4oT8/f01b9489enTp9jjeV/hQtx222264YYbVFhYqPHjxystLc2xj/cUXO3evVsvvviiJOk///mPUzeh0uJ9VTUwZgEog7MHdJ090MuTswd0nT3QC5XbkSNHNGjQIB05ckQ2m03vvfeehg0bVuI5vK9woYYNG6ZPPvlEmZmZWr58uWOgM+8puJo2bZry8vLUtGlTZWVlad68eW7H7Nixw/H4u+++09GjRyVJQ4cOVVhYGO+rKoKwAJRBmzZtHI93795d4rFn7/c0XSYqn+PHj+vyyy/X/v37JZ359m7UqFHnPI/3FS5UrVq1HI8PHjzoeNyiRQv5+/uroKCA9xQk/dktaP/+/br55pvPefw///lPx+MDBw4oLCyM31VVBN2QgDJo0qSJ6tevL8l57nxPvv/+e0lSgwYN1Lhx44ouDT526tQpXXHFFY55yKdMmaL77ruvVOfyvsKFOnz4sOPx2V09goKC1L17d0nS2rVrS+xfXvSeCw4OVteuXSuoUlQG/K6qGggLQBnYbDZHl5Ldu3frp59+8njcTz/95PhWZdiwYec1NR2sJysrS1dffbU2bdokSXryySf16KOPlvp83le4UAsWLHA8bteundO+4cOHS5JOnz6tzz//3OP5SUlJ+uabbyRJAwcOdOqTjsplzpw5MgyjxJ+zBz0nJCQ4thd92Od3VRVhAHA4cOCAIcmQZNx+++2lOmfPnj2Gv7+/Icno2rWrkZWV5bQ/KyvL6Nq1qyHJCAgIMH799dcKqBxmkZubawwePNjxPnrwwQcv6Dq8r3C2+Ph4Izs7u8RjXn31Vcf7rkmTJobdbnfan5qaakRFRRmSjEaNGhnHjx932m+3242hQ4c6rpGQkFDeLwMWM3ny5HO+H/hdVfkxZgFV2urVq7V3717H8+PHjzse79271236wdGjR7tdo0WLFvr73/+uKVOmaMOGDerZs6ceffRRNWvWTPv27dNLL72kzZs3S5L+/ve/66KLLqqQ1wJzuPnmm7VixQpJ0oABAzR27FinQYKugoKC1KJFC7ftvK9wtmeffVYTJ07U9ddfr169eqlZs2YKDw9Xenq6tm/fro8++khr1qyRdOY9NXPmTPn7+ztdIyYmRi+99JLuueceHTx4UJdccomefPJJtWvXTkeOHNFrr72mhIQESWfex/369fP2y4QF8buqCvB1WgF86fbbb3d8a1Kan+IUFBQYd9xxR4nnjh071igoKPDiq4MvnM/7Sf//DW9xeF+hSKNGjUr1foqNjTVWrFhR4rWeeeYZw2azFXuNIUOGnLMVA1VDaVoWDIPfVZUdYxaAcuDn56fZs2dr6dKlGjZsmOrXr6+goCDVr19fw4YN05dffql3331Xfn78Xw6lx/sKRb766iu98soruu6669S+fXvVqVNHAQEBioiIULNmzXT99dcrPj5ee/bs0eWXX17itZ577jmtXr1at9xyi+Li4hQUFKTatWvr8ssv19y5c7V06VKnxbaAc+F3VeVmMwzD8HURAAAAAMyHiAcAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAACo1J599lnZbDbZbDZflwIAlkNYAIAqKDEx0fEBuiw/AIDKjbAAAPCqOXPmOMJGYmKir8up1FauXOn4t165cqWvywFgQQG+LgAA4H0NGjTQ9u3bi93frl07SVLXrl0VHx/vrbIAACZDWACAKigwMFBt27Y953FhYWGlOg4AUDnRDQkAAACAR4QFAMB5Kyws1IcffqghQ4aobt26CgoKUq1atdS/f39Nnz5deXl5bucU9Z8fM2aMY1uTJk3cBk279q3/6aef9NRTT6lfv36Oe0VGRqpNmza69957tXPnzop+uQ7p6el65ZVXNGDAAKdaOnXqpPvvv19r1qwp9txjx47pqaeeUqdOnRQdHa2QkBA1btxYt912m1avXn3Oe3/33Xe6+eab1aRJE4WGhqpatWpq1KiRLr30Uj3yyCP67rvvHMcWDWDv37+/Y1v//v3d/q3nzJlTpn8PAFWAAQCAC0mGJKNv375u+1JTU42ePXs6jvH007p1ayMxMdHpvISEhBLPKfpJSEhwnBMfH3/O4/39/Y233nqr2NcyefJkx7Fl8fXXXxs1a9Y8Zz2efPXVV0ZkZGSJ5913331GQUGBx/MnTJhwzvvWqFHDcfyBAwdK9W8dHx9fpn8TAJUfYxYAAKVWUFCgv/zlL1q7dq0kqW/fvho/fryaNGmiI0eO6L333tPChQu1a9cuDRw4UFu2bFF4eLgkqVu3btq+fbsWLVqkp556SpL01VdfqX79+k73aNKkieOx3W5X9erVNWzYMPXp00cXXXSRwsLCdOTIEW3atElvvPGGjh8/rvHjx6tVq1YaMGBAhbzuhIQEXXXVVbLb7fL399dtt92mYcOGqWHDhsrJydHOnTu1bNkyLV682O3cLVu2aOjQocrLy1NgYKDGjx+va665RmFhYdq8ebOmTJmiAwcO6K233lJYWJheeuklp/OXLFmi1157TZLUvn173XvvvWrdurWioqJ08uRJ/fLLL/rmm2+0bt06xzlFA9jXr1+vO+64Q5L03nvvqVu3bk7Xjo2NLed/KQCVjq/TCgDAfFRMy8Kbb77p2Ddq1CijsLDQ7dwnnnjCccykSZPc9p/dWnDgwIES60hKSjIyMzOL3X/y5Emjffv2hiSjV69eHo8pa8tCdna2Ub9+fUOSUa1aNaeWD1e///6727Zu3bo5WkC++uort/1paWlGmzZtDEmGn5+fsWPHDqf9t912myHJaNSokZGenl7svVNTU922nd2aU1LdAFAcxiwAAErtrbfekiTVqlVLb775pseF2Z577jm1atVKkjRr1izl5uZe8P0aNGigatWqFbs/KipK//jHPyRJq1evVmpq6gXfqzj//e9/deTIEUnSCy+8oH79+hV7bFxcnNPzdevWaf369ZKku+66S4MHD3Y7p3r16po5c6akM2NBpk+f7rT/6NGjkqTOnTs7Wmk8iYmJOfeLAYDzRFgAAJTKkSNHtGvXLknSjTfeqIiICI/HBQQEOAYxnzhxQps2bSq3GjIzM5WYmKhffvlFO3bs0I4dOxQYGOjYv3Xr1nK7V5ElS5ZIOjON7F133XVe537zzTeOx2PHji32uJ49e6p169Zu50hSvXr1JEnff/+99u3bd173B4CyIiwAAEplx44djseXXHJJiceevf/s8y7E8ePH9cQTT6hly5aKiIhQkyZN1LZtW7Vr107t2rXT1Vdf7XRsedu8ebMkqUuXLiW2cnhS9NqDgoLUsWPHEo8t+jf77bffnGaTGjVqlCQpNTVVbdu21YgRIxQfH6+9e/eeVy0AcCEICwCAUklLS3M8rl27donH1q1b1+N552vjxo1q1aqVXnzxRf36668yDKPE47Ozsy/4XsUpCiBF3/Cfj6LXHhMTo4CAkucUKfo3MwxDJ06ccGwfOHCg3nzzTYWGhionJ0fz58/XHXfcoYsuukixsbG65557KqRFBQAkwgIA4AJ4GqtQ3vLy8nTjjTcqNTVVgYGBevjhh7Vq1SolJycrJydHhmHIMAynrjnnChO+UtZ/r/vuu0+JiYmaNm2ahgwZoqioKEnS4cOHNWPGDHXq1MkxwxQAlCfCAgCgVM4eQPvHH3+UeGzRoFzX887Hd999p/3790uSpk+frldeeUV9+vRR3bp1FRwc7DiuLC0XpVGzZk1JUnJy8nmfW/TaU1NTZbfbSzy26N/MZrOpevXqbvtr166tCRMmaOnSpUpLS9PGjRv11FNPKTo6WoZh6Pnnn9eiRYvOu0YAKAlhAQBQKm3btnU8/vnnn0s89uw5/88+Tyr9t+y//PKL4/FNN91U7HEbNmwo1fUuVOfOnR33ycrKOq9zi157Xl6etmzZUuKxRf9mF110kYKCgko81s/PT507d9Y///lPffvtt47tn3zyidNx3mgBAlC5ERYAAKVSv359x4w9n3zyiTIyMjweV1BQoDlz5kg6My1o0YftIiEhIY7HJU2revY38ZmZmR6PKSws1KxZs0pV/4UaOnSoJCkrK8sxxWlpDRo0yPH4vffeK/a4tWvXaufOnW7nlEbnzp0dLRGuA7xL+28NAMUhLAAASu2+++6TJB07dkwPPPCAx2Oee+45xwffu+66y6nLkOQ8ULikqUAvuugix+Oi8OHq8ccfL9epWT0ZOXKkGjRoIEl68skntWrVqmKPTUpKcnrevXt3de3aVdKZNSfObgUocurUKY0bN07SmRaDe++912n//PnzSxy4vWHDBseA6LNXv5ZK/28NAMWxGWYdDQYA8Jmi7it9+/bVypUrHdsLCgrUu3dvrV27VpI0YMAA/e1vf1OTJk2UnJys9957T59//rkkqVmzZtqyZYvbQmLp6emqXbu2cnJy1LlzZ02ZMkWNGjWSn9+Z768aNGig0NBQZWZmqmnTpkpJSZG/v7/uvPNOXXvttapZs6b27t3r+PDds2dPrVmzRpIUHx+v0aNHO93v2Wef1XPPPSfpwgdAJyQkaPDgwbLb7QoICNBtt92m4cOHKzY2Vrm5udq9e7e+/PJLffHFF27f4G/ZskWXXHKJ8vLyFBQUpPvvv19Dhw5VWFiYNm/erClTpjjGZkyaNEkvvfSS0/mNGzfWqVOnNGzYMPXp00ctWrRQWFiYUlNTtXr1av3nP/9RWlqa/P399dNPPznCSZG4uDglJSWpSZMmeu2119SyZUv5+/tLkurUqVPsehkAIEny2drRAADTkmRIMvr27eu2LzU11ejZs6fjGE8/rVu3NhITE4u9/qRJk4o9NyEhwXHc8uXLjZCQkGKP7devn7Fjxw7H8/j4eLd7TZ482bG/LJYvX25Ur169xNdd3D2++uorIzIyssTz7rvvPqOgoMDt3EaNGp3znsHBwR5fu2EYxvTp04s9r7hzAKAI3ZAAAOclJiZG33//vf773//qyiuvVJ06dRQYGKgaNWqoX79+evPNN7VlyxY1atSo2GtMmTJFs2bNUu/evRUTE+P4ptvVFVdcoQ0bNmjkyJGqX7++AgMDVatWLfXt21czZ87Ut99+q7CwsIp6qW617N+/Xy+88IJ69OihGjVqyN/fX5GRkercubMmTJjgNLD7bIMHD9bevXv1xBNPqGPHjoqMjFRwcLAaNmyoW2+9VT/88IPefPNNR+vK2RISEvT666/r+uuvV7t27VSrVi0FBAQoMjJSnTp10iOPPKKdO3e6tagUuffee/XZZ59p8ODBql279jnXewCAs9ENCQAAAIBHtCwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8Oj/ADrf3Kh2gSFUAAAAAElFTkSuQmCC" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(1, 1, figsize=(4, 3), dpi=200)\n", + "ax.plot(\n", + " cost_AGP[9:],\n", + " best_seen_AGP[9:],\n", + " label=\"AGP\"\n", + ")\n", + "ax.plot(\n", + " cost_MES[9:],\n", + " best_seen_MES[9:],\n", + " label=\"MES\"\n", + ")\n", + "\n", + "ax.set_title(\"Branin\", fontsize=\"12\")\n", + "ax.set_xlabel(\"Total cost\", fontsize=\"10\")\n", + "ax.set_ylabel(\"Best seen\", fontsize=\"10\")\n", + "ax.tick_params(labelsize=10)\n", + "ax.legend(loc=\"lower right\", fontsize=\"7\", frameon=True, ncol=1)\n", + "plt.tight_layout()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-12-18T11:41:02.192844700Z", + "start_time": "2023-12-18T11:41:01.912139300Z" + } + }, + "id": "e3604083ed32e0eb" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From b033d74592f1a74b043ab984fa3ef79fc813b81b Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 13:26:58 +0100 Subject: [PATCH 07/20] Add missing unit test for agp --- test/models/test_gp_regression_multisource.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/models/test_gp_regression_multisource.py b/test/models/test_gp_regression_multisource.py index 8bf57627d1..6bade820b2 100644 --- a/test/models/test_gp_regression_multisource.py +++ b/test/models/test_gp_regression_multisource.py @@ -100,6 +100,8 @@ def test_init_error(self): model = SingleTaskAugmentedGP(train_X, train_Y) self.assertIsInstance(model, SingleTaskAugmentedGP) + # Test initialization with m = 0 + self.assertRaises(InputDataError, SingleTaskAugmentedGP, train_X, train_Y, m=0) # Test initialization without true source points bounds = torch.stack([torch.zeros(d), torch.ones(d)]) bounds[0, -1] = 1 @@ -108,6 +110,8 @@ def test_init_error(self): train_X[:, -1] = torch.round(train_X[:, -1], decimals=0) self.assertRaises(InputDataError, SingleTaskAugmentedGP, train_X, train_Y) + + def test_get_reliable_observation(self): x = torch.linspace(0, 5, 15).reshape(-1, 1) true_y = torch.sin(x).reshape(-1, 1) @@ -373,6 +377,9 @@ def test_init_error(self): ) self.assertIsInstance(model, FixedNoiseAugmentedGP) + # Test initialization with m = 0 + self.assertRaises(InputDataError, FixedNoiseAugmentedGP, train_X, train_Y, + torch.full_like(train_Y, 0.01), m=0) # Test initialization without true source points bounds = torch.stack([torch.zeros(d), torch.ones(d)]) bounds[0, -1] = 1 From 5aeaa3d43ea9b8a6529f1ba1a66b93d082f8b603 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 13:31:09 +0100 Subject: [PATCH 08/20] Fix formatting issues --- test/models/test_gp_regression_multisource.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/models/test_gp_regression_multisource.py b/test/models/test_gp_regression_multisource.py index 6bade820b2..0c8ca03a9b 100644 --- a/test/models/test_gp_regression_multisource.py +++ b/test/models/test_gp_regression_multisource.py @@ -101,7 +101,9 @@ def test_init_error(self): self.assertIsInstance(model, SingleTaskAugmentedGP) # Test initialization with m = 0 - self.assertRaises(InputDataError, SingleTaskAugmentedGP, train_X, train_Y, m=0) + self.assertRaises( + InputDataError, SingleTaskAugmentedGP, train_X, train_Y, m=0 + ) # Test initialization without true source points bounds = torch.stack([torch.zeros(d), torch.ones(d)]) bounds[0, -1] = 1 @@ -110,8 +112,6 @@ def test_init_error(self): train_X[:, -1] = torch.round(train_X[:, -1], decimals=0) self.assertRaises(InputDataError, SingleTaskAugmentedGP, train_X, train_Y) - - def test_get_reliable_observation(self): x = torch.linspace(0, 5, 15).reshape(-1, 1) true_y = torch.sin(x).reshape(-1, 1) @@ -378,8 +378,14 @@ def test_init_error(self): self.assertIsInstance(model, FixedNoiseAugmentedGP) # Test initialization with m = 0 - self.assertRaises(InputDataError, FixedNoiseAugmentedGP, train_X, train_Y, - torch.full_like(train_Y, 0.01), m=0) + self.assertRaises( + InputDataError, + FixedNoiseAugmentedGP, + train_X, + train_Y, + torch.full_like(train_Y, 0.01), + m=0, + ) # Test initialization without true source points bounds = torch.stack([torch.zeros(d), torch.ones(d)]) bounds[0, -1] = 1 From 385843706a2cba6b7b87e6c1f16f225d0a438c12 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 15:33:13 +0100 Subject: [PATCH 09/20] Notebook description update --- tutorials/multi_source_bo.ipynb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tutorials/multi_source_bo.ipynb b/tutorials/multi_source_bo.ipynb index 93186bcf88..d7afb38dc4 100644 --- a/tutorials/multi_source_bo.ipynb +++ b/tutorials/multi_source_bo.ipynb @@ -5,9 +5,11 @@ "source": [ "## Multi-Information Source BO with Augmented Gaussian Processes\n", "\n", - "In this tutorial, we show how to perform multi-information source Bayesian optimization in BoTorch using the Augmented Gaussian Process (AGP) along with the Augmented UCB (AUCB) acquisition function [1]. The key idea of AGP is to fit a GP model for each information source and *augment* the GP of the high fidelity source with observations from the lower fidelities sources. Only observations considered as *reliable* are used to augment the GP. The UCB acquisition function is modified to take into account also the sources' cost.\n", + "In this tutorial, we show how to perform Multiple Information Source Bayesian Optimization in BoTorch based on the Augmented Gaussian Process (AGP) and the Augmented UCB (AUCB) acquisition function proposed in [1].\n", + "The key idea of the AGP is to fit a GP model for each information source and *augment* the observations on the high fidelity source with those from *cheaper* sources which can be considered as *reliable*. The GP model fitted on this *augmented* set of observations is the AGP.\n", + "The AUCB is a modification of the standard UCB -- computed on the AGP -- suitably proposed to also deal with the source-specific query cost.\n", "\n", - "We find the AGP performs well if compared with discrete multi-fidelity approaches [2].\n", + "We emprically show that the *AGP-based* Multiple Information Source Basyesian Optimization usually performs better than other multi-fidelity approaches [2].\n", "\n", "[1] [Candelieri, A., & Archetti, F. (2021). Sparsifying to optimize over multiple information sources: an augmented Gaussian process based algorithm. Structural and Multidisciplinary Optimization, 64, 239-255.](https://link.springer.com/article/10.1007/s00158-021-02882-7)\n", "[2] [The arxiv will be available soon.](https://arxiv.org/)\n" @@ -131,9 +133,11 @@ "cell_type": "markdown", "source": [ "### Problem setup\n", - "We'll consider the Augmented Branin multi-fidelity synthetic test problem. This function is a version of the Branin test function with an additional dimension representing the fidelity parameter. The function takes the form $f(x,s)$ where $x \\in [-5, 10] \\times [0, 15]$ and $s \\in [0,1]$. The target fidelity is 1.0, which means that our goal is to solve $\\max_x f(x,1.0)$ by making use of cheaper evaluations $f(x,s)$ for $s < 1.0$. In this example, we'll assume that the cost function takes the form $5.0 + s$, illustrating a situation where the fixed cost is $5.0$.\n", + "We consider the augmented Branin multi-fidelity synthetic test problem. It is important to clarify that *augmented* is not about the AGP: here, it has a different meaning. It means that the Branin test function has been modified by introducing an additional dimension representing the fidelity parameter.\n", "\n", - "Since a multi-information source context is considered, three different sources we'll be used with $s = 0.5, 0.75, 1.00$ respectively." + "The test function takes the form $f(x,s)$ where $x \\in [-5, 10] \\times [0, 15]$ and $s \\in [0,1]$. The target fidelity is 1.0, which means that our goal is to solve $\\max_x f(x,1.0)$ by making use of cheaper evaluations $f(x,s)$ for $s < 1.0$. In this example, we'll assume that the cost function takes the form $5.0 + s$, illustrating a situation where the fixed cost is $5.0$.\n", + "\n", + "Since a multiple information source context is considered, three different sources are considered, with $s = 0.5, 0.75, 1.00$, respectively." ], "metadata": { "collapsed": false From 2f72f5c4a48b41138d0626cb3717cef57467d560 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 18 Dec 2023 17:41:42 +0100 Subject: [PATCH 10/20] Fix title underline too short in docs --- sphinx/source/models.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/source/models.rst b/sphinx/source/models.rst index 6f5ffcac7a..38854678da 100644 --- a/sphinx/source/models.rst +++ b/sphinx/source/models.rst @@ -45,7 +45,7 @@ GP Regression Models :members: Multi Information Source GP Regression Models -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: botorch.models.gp_regression_multisource :members: From 545e3888417cdb33661e4efeea2168db641c1787 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Tue, 19 Dec 2023 10:34:14 +0100 Subject: [PATCH 11/20] Add AGP tutorial to tutorials.json --- website/tutorials.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/tutorials.json b/website/tutorials.json index 03d73e065d..57e7caabb9 100644 --- a/website/tutorials.json +++ b/website/tutorials.json @@ -140,6 +140,10 @@ "id": "discrete_multi_fidelity_bo", "title": "Multi-fidelity Bayesian optimization with discrete fidelities using KG" }, + { + "id": "multi_source_bo", + "title": "Multi-Information Source BO with Augmented Gaussian Processes" + }, { "id": "composite_bo_with_hogp", "title": "Composite Bayesian optimization with the High Order Gaussian Process" From e25856317de9b83373a8c310cf8afc60135e3dc5 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Thu, 21 Dec 2023 11:14:18 +0100 Subject: [PATCH 12/20] Fix error when using gpu --- botorch/acquisition/augmented_multisource.py | 2 +- test/models/test_gp_regression_multisource.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/botorch/acquisition/augmented_multisource.py b/botorch/acquisition/augmented_multisource.py index ee1dbc3fe7..ecb01d46b2 100644 --- a/botorch/acquisition/augmented_multisource.py +++ b/botorch/acquisition/augmented_multisource.py @@ -79,7 +79,7 @@ def forward(self, X: Tensor) -> Tensor: A `(b1 x ... bk)`-dim tensor of Augmented Upper Confidence Bound values at the given design points `X`. """ - alpha = torch.zeros(X.shape[0], dtype=X.dtype) + alpha = torch.zeros(X.shape[0], dtype=X.dtype, device=X.device) agp_mean, agp_sigma = self._mean_and_sigma(X[..., :-1]) cb = (self.best_f if self.maximize else -self.best_f) + ( (agp_mean if self.maximize else -agp_mean) + self.beta.sqrt() * agp_sigma diff --git a/test/models/test_gp_regression_multisource.py b/test/models/test_gp_regression_multisource.py index 0c8ca03a9b..285598c0ec 100644 --- a/test/models/test_gp_regression_multisource.py +++ b/test/models/test_gp_regression_multisource.py @@ -33,14 +33,13 @@ def _get_random_data_with_source(batch_shape, n, d, n_source, q=1, **tkwargs): - dtype = tkwargs.get("dtype", torch.float32) rep_shape = batch_shape + torch.Size([1, 1]) bounds = torch.stack([torch.zeros(d), torch.ones(d)]) bounds[-1, -1] = n_source - 1 train_x = ( - get_random_x_for_agp(n=n, bounds=bounds, q=q).repeat(rep_shape).type(dtype) + get_random_x_for_agp(n=n, bounds=bounds, q=q).repeat(rep_shape).to(**tkwargs) ) - train_y = torch.sin(train_x[..., :1] * (2 * math.pi)).type(dtype) + train_y = torch.sin(train_x[..., :1] * (2 * math.pi)).to(**tkwargs) train_y = train_y + 0.2 * torch.randn(n, 1, **tkwargs).repeat(rep_shape) return train_x, train_y From 9e11ebdfeb7b6ae6f963c32e9ee95e3a2c1ddbad Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Mon, 29 Jan 2024 09:27:04 +0100 Subject: [PATCH 13/20] Move contributions to community folders --- .../acquisition/augmented_multisource.py | 14 + .../models/gp_regression_multisource.py | 5 +- notebooks_community/multi_source_bo.ipynb | 711 ++++++++++++++++++ sphinx/source/acquisition.rst | 5 - sphinx/source/models.rst | 7 +- .../acquisition/test_multi_source.py | 4 +- .../models/test_gp_regression_multisource.py | 2 +- tutorials/multi_source_bo.ipynb | 708 ----------------- website/tutorials.json | 4 - 9 files changed, 732 insertions(+), 728 deletions(-) rename {botorch => botorch_community}/acquisition/augmented_multisource.py (94%) rename {botorch => botorch_community}/models/gp_regression_multisource.py (99%) create mode 100644 notebooks_community/multi_source_bo.ipynb rename {test => test_community}/acquisition/test_multi_source.py (97%) rename {test => test_community}/models/test_gp_regression_multisource.py (99%) delete mode 100644 tutorials/multi_source_bo.ipynb diff --git a/botorch/acquisition/augmented_multisource.py b/botorch_community/acquisition/augmented_multisource.py similarity index 94% rename from botorch/acquisition/augmented_multisource.py rename to botorch_community/acquisition/augmented_multisource.py index ecb01d46b2..fdfef202d1 100644 --- a/botorch/acquisition/augmented_multisource.py +++ b/botorch_community/acquisition/augmented_multisource.py @@ -4,6 +4,20 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +r""" +Multi-Source Upper Confidence Bound. + +References: + +.. [Ca2021ms] + Candelieri, A., & Archetti, F. (2021). + Sparsifying to optimize over multiple information sources: + an augmented Gaussian process based algorithm. + Structural and Multidisciplinary Optimization, 64, 239-255. + +Contributor: andreaponti5 +""" + from __future__ import annotations from typing import Dict, Optional, Tuple, Union diff --git a/botorch/models/gp_regression_multisource.py b/botorch_community/models/gp_regression_multisource.py similarity index 99% rename from botorch/models/gp_regression_multisource.py rename to botorch_community/models/gp_regression_multisource.py index 09fb3c0524..cb60438c08 100644 --- a/botorch/models/gp_regression_multisource.py +++ b/botorch_community/models/gp_regression_multisource.py @@ -7,14 +7,15 @@ r""" Multi-Source Gaussian Process Regression models based on GPyTorch models. -For more on Multi-Source BO, see the -`tutorial `__ +References: .. [Ca2021ms] Candelieri, A., & Archetti, F. (2021). Sparsifying to optimize over multiple information sources: an augmented Gaussian process based algorithm. Structural and Multidisciplinary Optimization, 64, 239-255. + +Contributor: andreaponti5 """ from __future__ import annotations diff --git a/notebooks_community/multi_source_bo.ipynb b/notebooks_community/multi_source_bo.ipynb new file mode 100644 index 0000000000..6475812b5d --- /dev/null +++ b/notebooks_community/multi_source_bo.ipynb @@ -0,0 +1,711 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "## Multi-Information Source BO with Augmented Gaussian Processes\n", + "- Contributors: andreaponti5\n", + "- Last updated: Jan 29, 2024\n", + "- BoTorch version: 0.9.5(dev)\n", + "\n", + "In this tutorial, we show how to perform Multiple Information Source Bayesian Optimization in BoTorch based on the Augmented Gaussian Process (AGP) and the Augmented UCB (AUCB) acquisition function proposed in [1].\n", + "The key idea of the AGP is to fit a GP model for each information source and *augment* the observations on the high fidelity source with those from *cheaper* sources which can be considered as *reliable*. The GP model fitted on this *augmented* set of observations is the AGP.\n", + "The AUCB is a modification of the standard UCB -- computed on the AGP -- suitably proposed to also deal with the source-specific query cost.\n", + "\n", + "We emprically show that the *AGP-based* Multiple Information Source Basyesian Optimization usually performs better than other multi-fidelity approaches [2].\n", + "\n", + "[1] [Candelieri, A., & Archetti, F. (2021). Sparsifying to optimize over multiple information sources: an augmented Gaussian process based algorithm. Structural and Multidisciplinary Optimization, 64, 239-255.](https://link.springer.com/article/10.1007/s00158-021-02882-7)\n", + "[2] [The arxiv will be available soon.](https://arxiv.org/)\n" + ], + "metadata": { + "collapsed": false + }, + "id": "826ca0dcff42a3ba" + }, + { + "cell_type": "code", + "execution_count": 1, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: matplotlib in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (3.8.2)\n", + "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (1.2.0)\n", + "Requirement already satisfied: cycler>=0.10 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (4.46.0)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (1.4.5)\n", + "Requirement already satisfied: numpy<2,>=1.21 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (1.26.0)\n", + "Requirement already satisfied: packaging>=20.0 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (23.2)\n", + "Requirement already satisfied: pillow>=8 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (10.1.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (3.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (2.8.2)\n", + "Requirement already satisfied: six>=1.5 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[notice] A new release of pip is available: 23.2.1 -> 23.3.2\n", + "[notice] To update, run: python.exe -m pip install --upgrade pip\n" + ] + } + ], + "source": [ + "!pip install matplotlib" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:04:03.628067200Z", + "start_time": "2024-01-29T08:03:59.110048Z" + } + }, + "id": "8aa9032dbb2c2b04" + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [], + "source": [ + "import os\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "from gpytorch import ExactMarginalLogLikelihood\n", + "\n", + "import botorch\n", + "from botorch import fit_gpytorch_mll\n", + "from botorch.acquisition import InverseCostWeightedUtility, qMultiFidelityMaxValueEntropy\n", + "from botorch_community.acquisition.augmented_multisource import AugmentedUpperConfidenceBound\n", + "from botorch.models import AffineFidelityCostModel, SingleTaskMultiFidelityGP\n", + "from botorch_community.models.gp_regression_multisource import SingleTaskAugmentedGP, get_random_x_for_agp\n", + "from botorch.models.transforms import Standardize\n", + "from botorch.optim import optimize_acqf, optimize_acqf_mixed\n", + "from botorch.test_functions.multi_fidelity import AugmentedBranin" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:04:09.639408100Z", + "start_time": "2024-01-29T08:04:03.628965400Z" + } + }, + "id": "e55defd1ee4a5b0f" + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "initial_id", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2024-01-29T08:04:09.678340900Z", + "start_time": "2024-01-29T08:04:09.664287600Z" + } + }, + "outputs": [], + "source": [ + "tkwargs = {\n", + " \"dtype\": torch.double,\n", + " \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n", + "}\n", + "SMOKE_TEST = os.environ.get(\"SMOKE_TEST\", False)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "outputs": [], + "source": [ + "N_ITER = 10 if SMOKE_TEST else 50\n", + "SEED = 3" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:04:09.679342100Z", + "start_time": "2024-01-29T08:04:09.669295Z" + } + }, + "id": "e316bd291459a135" + }, + { + "cell_type": "markdown", + "source": [ + "### Problem setup\n", + "We consider the augmented Branin multi-fidelity synthetic test problem. It is important to clarify that *augmented* is not about the AGP: here, it has a different meaning. It means that the Branin test function has been modified by introducing an additional dimension representing the fidelity parameter.\n", + "\n", + "The test function takes the form $f(x,s)$ where $x \\in [-5, 10] \\times [0, 15]$ and $s \\in [0,1]$. The target fidelity is 1.0, which means that our goal is to solve $\\max_x f(x,1.0)$ by making use of cheaper evaluations $f(x,s)$ for $s < 1.0$. In this example, we'll assume that the cost function takes the form $5.0 + s$, illustrating a situation where the fixed cost is $5.0$.\n", + "\n", + "Since a multiple information source context is considered, three different sources are considered, with $s = 0.5, 0.75, 1.00$, respectively." + ], + "metadata": { + "collapsed": false + }, + "id": "6b58c67f5dbf329c" + }, + { + "cell_type": "code", + "execution_count": 5, + "outputs": [], + "source": [ + "problem = AugmentedBranin(negate=True).to(**tkwargs)\n", + "fidelities = torch.tensor([0.5, 0.75, 1.0], **tkwargs)\n", + "n_sources = fidelities.shape[0]\n", + "\n", + "bounds = torch.tensor([[-5, 0, 0], [10, 15, n_sources - 1]], **tkwargs)\n", + "target_fidelities = {n_sources - 1: 1.0}\n", + "\n", + "cost_model = AffineFidelityCostModel(fidelity_weights=target_fidelities, fixed_cost=5.0)\n", + "cost_aware_utility = InverseCostWeightedUtility(cost_model=cost_model)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:04:09.761367100Z", + "start_time": "2024-01-29T08:04:09.677340600Z" + } + }, + "id": "5f13380e681011ea" + }, + { + "cell_type": "markdown", + "source": [ + "### Model initialization\n", + "\n", + "We use a `SingleTaskAugmentedGP` to implement our AGP.\n", + "\n", + "At each Bayesian Optimization iteration, the set of observations from the *ground-truth* (i.e., the highest fidelity and more expensive source) is temporarily *augmented* by including observations from the other cheap sources, only if they can be considered *reliable*. Specifically, an observation $(x,y)$ from a cheap source is considered reliable if it satisfies the following inequality:\n", + "\n", + "$$\\vert\\mu(x)-y\\vert \\leq m \\sigma(x)$$\n", + "\n", + "where $\\mu(x)$ and $\\sigma(x)$ are, respectively, the posterior mean and standard deviation of the GP model fitted on the high fidelity observations only, and $m$ is a technical parameter making more *conservative* ($m→0$) or *inclusive* ($m→∞)$ the augmentation process. As reported in [1], a suitable value for this parameter is $m=1$.\n", + "\n", + "After the set of observations is augmented, the AGP is fitted through `SingleTaskAugmentedGP`.\n" + ], + "metadata": { + "collapsed": false + }, + "id": "81e30344694a9583" + }, + { + "cell_type": "code", + "execution_count": 6, + "outputs": [], + "source": [ + "def generate_initial_data(n):\n", + " train_x = get_random_x_for_agp(n, bounds, 1)\n", + " xs = train_x[..., :-1]\n", + " fids = fidelities[train_x[..., -1].int()].reshape(-1, 1)\n", + " train_obj = problem(torch.cat((xs, fids), dim=1)).unsqueeze(-1)\n", + " return train_x, train_obj\n", + "\n", + "\n", + "def initialize_model(train_x, train_obj, m):\n", + " model = SingleTaskAugmentedGP(\n", + " train_x, train_obj, m=m, outcome_transform=Standardize(m=1),\n", + " )\n", + " mll = ExactMarginalLogLikelihood(model.likelihood, model)\n", + " return mll, model" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:04:09.772956200Z", + "start_time": "2024-01-29T08:04:09.765101800Z" + } + }, + "id": "f8272160f69227ef" + }, + { + "cell_type": "markdown", + "source": [ + "#### Define a helper function that performs the essential BO step\n", + "This helper function optimizes the acquisition function and returns the candidate point along with the observed function values.\n", + "\n", + "The UCB acquisition function has been modified to deal with both the *discrepancy* between information sources and the *source-specific query cost*.\n", + "\n", + "Formally, the AUCB acquisition function, at a generic iteration $t$, is defined as:\n", + "\n", + "$$\\alpha_s(x,\\hat y^+) = \\frac{\\left[\\hat{\\mu}(x) + \\sqrt{\\beta^{(t)}} \\hat{\\sigma}(x)\\right] - \\hat{y}^+}{c_s \\cdot (1+\\vert \\hat{\\mu}(x) - \\mu_s(x) \\vert)} $$\n", + "\n", + "where $\\hat{y}^+$ is the best (i.e., highest) value in the *augmented* set of observations, the numerator is -- therefore -- the optimistic improvement with respect to $\\hat{y}^+$, $c_s$ is the query cost for the source $s$, and $\\vert \\hat{\\mu}(x) - \\mu_s(x) \\vert$ is a discrepancy measure between the predictions provided by the AGP and the GP on the source $s$, respectively, given the input $x$ (i.e., 1 is added just to avoid division by zero).\n", + "\n", + "For more information, please refer to [1]," + ], + "metadata": { + "collapsed": false + }, + "id": "ad21cd7999805d78" + }, + { + "cell_type": "code", + "execution_count": 7, + "outputs": [], + "source": [ + "def optimize_aucb(acqf):\n", + " candidate, value = optimize_acqf(\n", + " acq_function=acqf,\n", + " bounds=bounds,\n", + " q=1,\n", + " num_restarts=5,\n", + " raw_samples=128,\n", + " )\n", + " # observe new values\n", + " new_x = candidate.detach()\n", + " new_x[:, -1] = torch.round(new_x[:, -1], decimals=0)\n", + " return new_x" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:04:09.783470800Z", + "start_time": "2024-01-29T08:04:09.769617200Z" + } + }, + "id": "311309bd6a4d3a92" + }, + { + "cell_type": "markdown", + "source": [ + "### Perform a few steps of multi-fidelity BO\n", + "First, let's generate some initial random data and fit a surrogate model." + ], + "metadata": { + "collapsed": false + }, + "id": "667b55ca7ae58af3" + }, + { + "cell_type": "code", + "execution_count": 8, + "outputs": [], + "source": [ + "torch.manual_seed(SEED)\n", + "train_x, train_obj = generate_initial_data(n=5)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:04:11.943091700Z", + "start_time": "2024-01-29T08:04:09.778471900Z" + } + }, + "id": "a2b9dbfbae2f7d5" + }, + { + "cell_type": "markdown", + "source": [ + "We can now use the helper functions above to run a few iterations of BO." + ], + "metadata": { + "collapsed": false + }, + "id": "ca54230c1481ba60" + }, + { + "cell_type": "code", + "execution_count": 9, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iter 0;\t Fid = 1.00;\t Obj = -12.0999;\n", + "Iter 1;\t Fid = 1.00;\t Obj = -50.0743;\n", + "Iter 2;\t Fid = 0.50;\t Obj = -11.9672;\n", + "Iter 3;\t Fid = 0.50;\t Obj = -8.7046;\n", + "Iter 4;\t Fid = 0.50;\t Obj = -13.3425;\n", + "Iter 5;\t Fid = 1.00;\t Obj = -11.6850;\n", + "Iter 6;\t Fid = 0.50;\t Obj = -8.2926;\n", + "Iter 7;\t Fid = 1.00;\t Obj = -14.4455;\n", + "Iter 8;\t Fid = 0.50;\t Obj = -16.5711;\n", + "Iter 9;\t Fid = 1.00;\t Obj = -32.7208;\n", + "Iter 10;\t Fid = 0.50;\t Obj = -12.7833;\n", + "Iter 11;\t Fid = 1.00;\t Obj = -40.2770;\n", + "Iter 12;\t Fid = 0.50;\t Obj = -4.8935;\n", + "Iter 13;\t Fid = 1.00;\t Obj = -15.2650;\n", + "Iter 14;\t Fid = 0.50;\t Obj = -6.9511;\n", + "Iter 15;\t Fid = 1.00;\t Obj = -18.1231;\n", + "Iter 16;\t Fid = 0.50;\t Obj = -0.6015;\n", + "Iter 17;\t Fid = 1.00;\t Obj = -17.0770;\n", + "Iter 18;\t Fid = 0.50;\t Obj = -4.3991;\n", + "Iter 19;\t Fid = 1.00;\t Obj = -12.4392;\n", + "Iter 20;\t Fid = 1.00;\t Obj = -112.9747;\n", + "Iter 21;\t Fid = 0.50;\t Obj = -4.6979;\n", + "Iter 22;\t Fid = 0.50;\t Obj = -13.1965;\n", + "Iter 23;\t Fid = 1.00;\t Obj = -14.2637;\n", + "Iter 24;\t Fid = 0.50;\t Obj = -16.4335;\n", + "Iter 25;\t Fid = 1.00;\t Obj = -1.4522;\n", + "Iter 26;\t Fid = 0.50;\t Obj = -14.7771;\n", + "Iter 27;\t Fid = 1.00;\t Obj = -14.4608;\n", + "Iter 28;\t Fid = 1.00;\t Obj = -15.3493;\n", + "Iter 29;\t Fid = 1.00;\t Obj = -27.3031;\n", + "Iter 30;\t Fid = 0.50;\t Obj = -13.6875;\n", + "Iter 31;\t Fid = 0.50;\t Obj = -9.9730;\n", + "Iter 32;\t Fid = 0.50;\t Obj = -9.1073;\n", + "Iter 33;\t Fid = 1.00;\t Obj = -20.4122;\n", + "Iter 34;\t Fid = 1.00;\t Obj = -11.3355;\n", + "Iter 35;\t Fid = 0.50;\t Obj = -31.8170;\n", + "Iter 36;\t Fid = 0.50;\t Obj = -9.0516;\n", + "Iter 37;\t Fid = 1.00;\t Obj = -6.1756;\n", + "Iter 38;\t Fid = 0.50;\t Obj = -4.9320;\n", + "Iter 39;\t Fid = 1.00;\t Obj = -7.7986;\n", + "Iter 40;\t Fid = 0.50;\t Obj = -8.6059;\n", + "Iter 41;\t Fid = 1.00;\t Obj = -17.1693;\n", + "Iter 42;\t Fid = 0.50;\t Obj = -10.7690;\n", + "Iter 43;\t Fid = 1.00;\t Obj = -25.7958;\n", + "Iter 44;\t Fid = 1.00;\t Obj = -122.8804;\n", + "Iter 45;\t Fid = 1.00;\t Obj = -8.2496;\n", + "Iter 46;\t Fid = 0.50;\t Obj = -0.7404;\n", + "Iter 47;\t Fid = 1.00;\t Obj = -13.2155;\n", + "Iter 48;\t Fid = 1.00;\t Obj = -0.4122;\n", + "Iter 49;\t Fid = 0.50;\t Obj = -0.4802;\n" + ] + } + ], + "source": [ + "cumulative_cost = 0.0\n", + "\n", + "with botorch.settings.validate_input_scaling(False):\n", + " for it in range(N_ITER):\n", + " mll, model = initialize_model(train_x, train_obj, m=1)\n", + " fit_gpytorch_mll(mll)\n", + " acqf = AugmentedUpperConfidenceBound(\n", + " model,\n", + " beta=3,\n", + " maximize=True,\n", + " best_f=train_obj[torch.where(train_x[:, -1] == 0)].min(),\n", + " cost={i: fid + 5.0 for i, fid in enumerate(fidelities)},\n", + " )\n", + " new_x = optimize_aucb(acqf)\n", + " if model.n_true_points < model.max_n_cheap_points:\n", + " new_x[:, -1] = fidelities.shape[0] - 1\n", + " train_x = torch.cat([train_x, new_x])\n", + "\n", + " new_x[:, -1] = fidelities[new_x[:, -1].int()]\n", + " new_obj = problem(new_x).unsqueeze(-1)\n", + " train_obj = torch.cat([train_obj, new_obj])\n", + "\n", + " print(\n", + " f\"Iter {it};\"\n", + " f\"\\t Fid = {new_x[0].tolist()[-1]:.2f};\"\n", + " f\"\\t Obj = {new_obj[0][0].tolist():.4f};\"\n", + " )" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:08:22.793747600Z", + "start_time": "2024-01-29T08:04:11.940091600Z" + } + }, + "id": "8d02e319798a28d7" + }, + { + "cell_type": "markdown", + "source": [ + "## Comparison to MES" + ], + "metadata": { + "collapsed": false + }, + "id": "ab5e775efb0ccfe9" + }, + { + "cell_type": "code", + "execution_count": 10, + "outputs": [], + "source": [ + "def initialize_mes_model(train_x, train_obj, data_fidelity):\n", + " model = SingleTaskMultiFidelityGP(\n", + " train_x,\n", + " train_obj,\n", + " outcome_transform=Standardize(m=1),\n", + " data_fidelity=data_fidelity,\n", + " )\n", + " mll = ExactMarginalLogLikelihood(model.likelihood, model)\n", + " return mll, model" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:08:22.804418600Z", + "start_time": "2024-01-29T08:08:22.796748900Z" + } + }, + "id": "1c93f20bdaffccac" + }, + { + "cell_type": "code", + "execution_count": 11, + "outputs": [], + "source": [ + "def optimize_mes_and_get_observation(mes_acq, fixed_features_list):\n", + " candidates, acq_value = optimize_acqf_mixed(\n", + " acq_function=mes_acq,\n", + " bounds=problem.bounds,\n", + " q=1,\n", + " num_restarts=5,\n", + " raw_samples=128,\n", + " fixed_features_list=fixed_features_list,\n", + " )\n", + " # observe new values\n", + " cost = cost_model(candidates).sum()\n", + " new_x = candidates.detach()\n", + " new_obj = problem(new_x).unsqueeze(-1)\n", + " return new_x, new_obj, cost" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:08:22.837919300Z", + "start_time": "2024-01-29T08:08:22.806683100Z" + } + }, + "id": "1a1bba8e3496b54d" + }, + { + "cell_type": "code", + "execution_count": 12, + "outputs": [], + "source": [ + "train_x_mes = torch.clone(train_x[:10])\n", + "train_x_mes[:, -1] = fidelities[train_x_mes[:, -1].int()]\n", + "train_obj_mes = torch.clone(train_obj[:10])" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:08:22.840918900Z", + "start_time": "2024-01-29T08:08:22.819913500Z" + } + }, + "id": "e8414dfbd0643afa" + }, + { + "cell_type": "code", + "execution_count": 13, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iter 0;\t Fid = 0.50;\t Obj = -8.3276;\n", + "Iter 1;\t Fid = 0.50;\t Obj = -21.1510;\n", + "Iter 2;\t Fid = 0.50;\t Obj = -20.7855;\n", + "Iter 3;\t Fid = 0.50;\t Obj = -8.5067;\n", + "Iter 4;\t Fid = 0.50;\t Obj = -49.7241;\n", + "Iter 5;\t Fid = 0.50;\t Obj = -1.3525;\n", + "Iter 6;\t Fid = 0.50;\t Obj = -98.0777;\n", + "Iter 7;\t Fid = 0.50;\t Obj = -88.9142;\n", + "Iter 8;\t Fid = 0.50;\t Obj = -1.3561;\n", + "Iter 9;\t Fid = 1.00;\t Obj = -135.8402;\n", + "Iter 10;\t Fid = 0.50;\t Obj = -16.4201;\n", + "Iter 11;\t Fid = 0.50;\t Obj = -125.3250;\n", + "Iter 12;\t Fid = 0.50;\t Obj = -6.7582;\n", + "Iter 13;\t Fid = 0.50;\t Obj = -61.3218;\n", + "Iter 14;\t Fid = 0.50;\t Obj = -19.6356;\n", + "Iter 15;\t Fid = 0.50;\t Obj = -91.3633;\n", + "Iter 16;\t Fid = 0.50;\t Obj = -90.9424;\n", + "Iter 17;\t Fid = 0.50;\t Obj = -22.3466;\n", + "Iter 18;\t Fid = 0.50;\t Obj = -1.9025;\n", + "Iter 19;\t Fid = 0.50;\t Obj = -16.0446;\n", + "Iter 20;\t Fid = 0.50;\t Obj = -13.2516;\n", + "Iter 21;\t Fid = 0.50;\t Obj = -23.5304;\n", + "Iter 22;\t Fid = 0.50;\t Obj = -8.6583;\n", + "Iter 23;\t Fid = 0.50;\t Obj = -19.9776;\n", + "Iter 24;\t Fid = 0.50;\t Obj = -22.8034;\n", + "Iter 25;\t Fid = 0.50;\t Obj = -11.0479;\n", + "Iter 26;\t Fid = 0.50;\t Obj = -1.0324;\n", + "Iter 27;\t Fid = 0.50;\t Obj = -3.1515;\n", + "Iter 28;\t Fid = 0.50;\t Obj = -20.1842;\n", + "Iter 29;\t Fid = 0.50;\t Obj = -4.3885;\n", + "Iter 30;\t Fid = 0.50;\t Obj = -262.8294;\n", + "Iter 31;\t Fid = 0.50;\t Obj = -13.1549;\n", + "Iter 32;\t Fid = 0.50;\t Obj = -35.3697;\n", + "Iter 33;\t Fid = 0.50;\t Obj = -9.8748;\n", + "Iter 34;\t Fid = 0.50;\t Obj = -5.5868;\n", + "Iter 35;\t Fid = 0.50;\t Obj = -262.2982;\n", + "Iter 36;\t Fid = 1.00;\t Obj = -1.1024;\n", + "Iter 37;\t Fid = 0.50;\t Obj = -1.9896;\n", + "Iter 38;\t Fid = 1.00;\t Obj = -10.5575;\n", + "Iter 39;\t Fid = 1.00;\t Obj = -8.7057;\n", + "Iter 40;\t Fid = 1.00;\t Obj = -2.3179;\n", + "Iter 41;\t Fid = 1.00;\t Obj = -1.9847;\n", + "Iter 42;\t Fid = 1.00;\t Obj = -6.1164;\n", + "Iter 43;\t Fid = 1.00;\t Obj = -6.2065;\n", + "Iter 44;\t Fid = 0.50;\t Obj = -110.4581;\n", + "Iter 45;\t Fid = 1.00;\t Obj = -2.2320;\n", + "Iter 46;\t Fid = 0.50;\t Obj = -256.6816;\n", + "Iter 47;\t Fid = 1.00;\t Obj = -7.6145;\n", + "Iter 48;\t Fid = 0.75;\t Obj = -15.1647;\n", + "Iter 49;\t Fid = 1.00;\t Obj = -4.1053;\n" + ] + } + ], + "source": [ + "candidate_set = torch.rand(\n", + " 1000, problem.bounds.size(1), device=problem.bounds.device, dtype=problem.bounds.dtype\n", + ")\n", + "candidate_set = problem.bounds[0] + (problem.bounds[1] - problem.bounds[0]) * candidate_set\n", + "\n", + "cumulative_cost = 0.0\n", + "\n", + "with botorch.settings.validate_input_scaling(False):\n", + " for it in range(N_ITER):\n", + " mll, model = initialize_mes_model(train_x_mes, train_obj_mes, data_fidelity=2)\n", + " fit_gpytorch_mll(mll)\n", + " acqf = qMultiFidelityMaxValueEntropy(\n", + " model, candidate_set, cost_aware_utility=cost_aware_utility\n", + " )\n", + " new_x, new_obj, cost = optimize_mes_and_get_observation(acqf,\n", + " fixed_features_list=[{2: fid} for fid in fidelities])\n", + " train_x_mes = torch.cat([train_x_mes, new_x])\n", + " train_obj_mes = torch.cat([train_obj_mes, new_obj])\n", + " cumulative_cost += cost\n", + " print(\n", + " f\"Iter {it};\"\n", + " f\"\\t Fid = {new_x[0].tolist()[-1]:.2f};\"\n", + " f\"\\t Obj = {new_obj[0][0].tolist():.4f};\"\n", + " )" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:15:52.142811900Z", + "start_time": "2024-01-29T08:08:22.829920500Z" + } + }, + "id": "fcaf20bf41f1b680" + }, + { + "cell_type": "markdown", + "source": [ + "## Plot results" + ], + "metadata": { + "collapsed": false + }, + "id": "dd44be2238fc0110" + }, + { + "cell_type": "code", + "execution_count": 14, + "outputs": [], + "source": [ + "mapping_fid = dict(zip(range(fidelities.shape[0]), fidelities.tolist()))\n", + "cost_AGP = torch.cumsum(torch.tensor([mapping_fid[int(source)] for source in train_x[:, -1].tolist()]), dim=0)\n", + "cost_MES = torch.cumsum(train_x_mes[:, -1], dim=0)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:15:52.144810Z", + "start_time": "2024-01-29T08:15:52.137410200Z" + } + }, + "id": "5c5cc1ef1808ad1f" + }, + { + "cell_type": "code", + "execution_count": 15, + "outputs": [], + "source": [ + "train_obj[torch.where(train_x[:, -1] != fidelities.shape[0] - 1)] = train_obj.min()\n", + "best_seen_AGP = torch.cummax(train_obj, dim=0)[0]" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:15:52.159128900Z", + "start_time": "2024-01-29T08:15:52.145813Z" + } + }, + "id": "4a7f7020f195a31f" + }, + { + "cell_type": "code", + "execution_count": 16, + "outputs": [], + "source": [ + "train_obj_mes[torch.where(train_x_mes[:, -1] != 1)[0]] = train_obj_mes.min()\n", + "best_seen_MES = torch.cummax(train_obj_mes, dim=0)[0]" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:15:52.162132Z", + "start_time": "2024-01-29T08:15:52.155612900Z" + } + }, + "id": "f696548b9b3f50a5" + }, + { + "cell_type": "code", + "execution_count": 23, + "outputs": [ + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwsAAAJECAYAAABZ37i3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAB7CAAAewgFu0HU+AABs9klEQVR4nO3dd3xUVf7/8fekk0BCr6GEJqCoVF16ExUXAV0VERBExYKKDeuK7n5VlFWsKCIkNqQoCwKCWAIK0psgRakSCQZCTWeS+/uDX2aZkgLJzNy583o+Hjx2cusZnL3kPed8zrEZhmEIAAAAAFyE+LsBAAAAAMyJsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAB4WY8ePWSz2WSz2bRs2TJ/NwcASo2wAAA4L+f+4uvpT0hIiCpVqqSEhAQNHDhQU6dO1enTp/3dbADABSAsAADKlWEYysjI0P79+zV//nzdfffdatasmb766it/Nw0AcJ7C/N0AAEDg6tChgzp27Oi0raCgQCdOnNCWLVu0fft2SdJff/2lG264QfPmzdPf//53fzQVAHABCAsAgAvWr18/Pf/880XuX7lypQYPHqyUlBTl5+frnnvu0b59+xQeHu67RpoAdQoAAhXDkAAAXtO5c2fNmTPH8fOff/7JL84AEEAICwAAr7ryyiuVkJDg+LlwaBIAwPwICwAAr6tTp47jdWZmptv+/fv3O2ZTatSokWP7ihUrdOedd6pFixaKi4uTzWbT2LFjnc4tKCjQTz/9pOeee059+/ZVgwYNFB0drcjISNWpU0e9evXSiy++qKNHj5aqrefO7FRo165dGjt2rFq2bKmKFSsqNjZWl112mZ566qlSXbc0U6eOGDHCcUxSUpIkKSsrS5MnT1aXLl1Uq1YtRUZGqn79+rr11lu1cuXKUr0fACgLahYAAF53+PBhx+vatWuXeHxeXp4efPBBTZkypdjjzpw5o4SEBP35559F3vfw4cNKTk7Wyy+/rPfff19Dhw49r7a///77Gjt2rHJzc522//LLL/rll180depULVmyRO3btz+v65Zk+/bt+sc//qEdO3Y4bU9JSdHMmTM1c+ZMPffcc3rhhRfK9b4AcC7CAgDAq9avX6+9e/c6fu7atWuJ5zz88MOOoNC6dWtddtllCg8P12+//aaQkP91iufn5zuCQsWKFXXxxRercePGio2N1ZkzZ5SSkqLVq1fr1KlTyszM1LBhwxQeHq5bbrmlVG1PSkrSvffeK0m66KKL1L59e1WoUEE7d+7UypUrZRiG0tPTdf3112vHjh2Ki4sr9d9LcQ4dOqQ+ffooNTVVlStXVteuXVW7dm0dPXpUP/zwg06ePClJ+te//qVWrVqV+v0AwHkzAAA4D927dzckGZKM8ePHF3vs2rVrjUaNGjmOHzRokMfj9u3b5zgmNDTUkGTUr1/f+PHHH92OzcnJcbzOzc01Ro4caSQnJxt5eXker52Tk2O8+uqrRlhYmCHJqFy5snH69Oki21zYDklGZGSkUaNGDWPx4sVuxy1fvtyIjY11HPvCCy8Uec1z/86Sk5M9HnP77bc73VeS8cQTTxiZmZlOx6Wnpxu9evVyHNu4cWOjoKCgyHsDQFnQswAAuGBff/2125j9goICnTx5Ur/88ou2bdvm2D5o0CB9+umnJV4zPz9f0dHR+u6779S8eXO3/ZGRkY7XERERmj59erHXi4yM1OOPP66CggI9+eSTOnHihD755BNHj0FJvvvuO1166aVu27t166aXXnpJY8aMkSR9/vnneu6550p1zZLk5ubqqaee0ksvveS2r2rVqpoxY4aaNGmizMxM7d27V2vXrtUVV1xRLvcGgHMRFgAAF2zdunVat25dscfUqVNHkydP1sCBA0t93TFjxngMCmUxcuRIPfnkk5LOBoDShIW7777bY1AoNHz4cI0dO1Z2u127du3SqVOnFBsbW+a21qhRo9jgUatWLV133XWaPXu2JBEWAHgNYQEA4FWpqam68cYbNWTIEL311luqUqVKiecMHjz4vO9TUFCgDRs2aPPmzUpJSdGpU6d05swZj8du3ry5VNe86aabit1fqVIlNWnSRLt27ZJhGDpw4IBat259vk13079/f0VFRRV7TJs2bRxhYf/+/WW+JwB4QlgAAFyw8ePHe1zBOTMzU/v379fixYv16quv6siRI/r000+1adMm/fTTT8UGhvDw8PP6hdtut+utt97SpEmTlJKSUqpzSjuNamnaUa1aNcfrU6dOleq6Zr0vALhinQUAQLmLiYnRxRdfrMcee0ybNm1SvXr1JEm//vqrHnnkkWLPrVKlisLCSvddVm5urq677jo9+uijpQ4KknT69OlSHVea2Y3Cw8Mdr4vqyThf/rovALgiLAAAvKpevXoaP3684+dPP/3Uad0FVxUqVCj1tV944QUtXbpU0tnF1G655RbNnj1bO3bs0MmTJ5WXlyfDMBx/Cp37ujjnLszmS/66LwC4YhgSAMDrrr76asdru92u5cuXl3ltgNzcXL399tuOn5OSkjR8+PAijy9tbwIA4H/oWQAAeF2dOnWcfj5w4ECZr7l27VplZGRIki6++OJig0J53RMAgg1hAQDgdVlZWU4/n7sK84U6dOiQ43VpCoJ//PHHMt8TAIINYQEA4HUbN250+rmw4Lkszg0crmHEVUFBgT744IMy3xMAgg1hAQDgdZMmTXK8ttls6tWrV5mv2bhxY8fr5cuX6+TJk0UeO3HiRG3ZsqXM9wSAYENYAAB4zYkTJzR69GgtWLDAsW3IkCGqVatWma/dpk0bRw/FyZMnddNNNzkNTZLOFkE/99xzevLJJxUTE1PmewJAsGE2JADABfv66689LnCWlZWl/fv3a/Xq1crOznZsb968uV5//fVyuXdISIj+/e9/64477pAkffvtt2revLk6deqkhg0bKj09XcuWLdPx48clSR988IFuu+22crk3AAQLwgIA4IKtW7dO69atK9Wx119/vaZMmaKaNWuW2/1Hjhyp3bt366WXXpJ0duXob7/91umYqKgovfHGGxoyZAhhAQDOE2EBAFDuIiMjFRcXp6ZNm+rKK6/UkCFD1K5dO6/c68UXX9S1116rd955RytWrNCRI0dUqVIlxcfH65prrtGoUaPUrFkzr9wbAKzOZpR2GUsAAAAAQYUCZwAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEhQBw4cECPPvqoWrRooZiYGFWtWlUdOnTQxIkTlZWV5e/mAQAAwIJshmEY/m4EirdgwQINHTpUp06d8ri/efPmWrRokZo2berjlgEAAMDKCAsmt2nTJnXu3FnZ2dmqWLGinnrqKfXs2VPZ2dmaOXOmpk6dKulsYFi/fr0qVark5xYDAADAKggLJtetWzf99NNPCgsL048//qi//e1vTvsnTpyocePGSZLGjx+v559/vtzunZOTo61bt0qSatSoobCwsHK7NgAAAMqX3W7XkSNHJEmtW7dWVFRUma9JWDCxtWvX6oorrpAkjR49Wu+//77bMQUFBbrkkku0Y8cOVa5cWWlpaQoPDy+X+69bt04dO3Ysl2sBAADAd9auXasOHTqU+ToUOJvYvHnzHK9Hjhzp8ZiQkBANHz5cknTixAklJyf7omkAAAAIAowrMbEVK1ZIkmJiYtSuXbsij+vevbvj9cqVK9W3b99yuX+NGjUcr9euXas6deqUy3UBAABQ/lJTUx2jQs79Pa4sCAsmtmPHDklS06ZNi60XaNGihds55eHce9apU0fx8fHldm0AAAB4T3nVmhIWTConJ0dHjx6VpBJ/Sa9SpYpiYmKUmZmpgwcPlvoeKSkpxe5PTU0t9bUAAABgPYQFkzp9+rTjdcWKFUs8vjAsZGRklPoe9evXv6C2AQAAIDhQ4GxSOTk5jtcRERElHh8ZGSlJys7O9lqbAAAAEFzoWTCpc+fFzcvLK/H43NxcSVKFChVKfY+ShiydWyQDAACA4ENYMKlzV2IuzdCizMxMSaUbslSIgmUAAAAUh2FIJhUVFaVq1apJKrkQ+fjx446wQB0CAAAAygthwcRatWolSdq9e7fsdnuRx+3cudPxumXLll5vFwAAAIIDYcHEunTpIunsEKMNGzYUedzy5csdrzt37uz1dgEAACA4EBZMbODAgY7XiYmJHo8pKCjQxx9/LEmqXLmyevbs6YumAQAAIAgQFkysY8eO6tq1qyRp2rRpWrVqldsxr732mmPV5oceekjh4eE+bSMAAACsi9mQTO7NN99U586dlZ2drb59++rpp59Wz549lZ2drZkzZ+qDDz6QJDVv3lyPPvqon1sLAAAAKyEsmFybNm00a9YsDR06VKdOndLTTz/tdkzz5s21aNEip+lWAQAAgLJiGFIA6N+/v3755Rc9/PDDat68uaKjo1W5cmW1b99er7zyijZt2qSmTZv6u5kAAACwGJthGIa/GwFzSklJcazbcPDgQRZxAwAA+P8Mw9Bna/7Qt9v/Uq49v8Tjm9eqpH8NuMSrbfLG724MQwIAAADO08x1B/XsvG2lPj7PXuDF1ngPw5AAAACA85BzJl9vfPebv5vhE4QFAAAA4DzMWX9Qf53K9XczfIJhSAAAAEAp5drzNXnZHqdtl8XH6Ya2xdcH1KgU6c1meQ1hAQAAACilLzakKPVkjtO2p/u11BWNq/mpRd7FMCQAAACgFPLsBZqc7NyrcEVCVcsGBYmwAAAAAJTK3I0p+vNEttO2h3o381NrfIOwAAAAAJTgTH6B3kne7bStQ6Mq+lsT6/YqSIQFAAAAoET/3fSnUo479yo82LuZbDabn1rkG4QFAAAAoBj2/AK969Kr0LZBZXVpWt1PLfIdwgIAAABQjPmbD+lAepbTtmDoVZAICwAAAECR8gsMt1qFy+pXVvfmNfzUIt8iLAAAAABFWLDlkPYdzXTa9lDvpkHRqyARFgAAAACP8gsMvf3D707bWteLU8+LavqpRb5HWAAAAAA8WLQ1VXuOOPcqBEutQiHCAgAAAOCioMDQ29879yq0qhOrPi2Dp1dBIiwAAAAAbhZvO6zf0zKctgVbr4JEWAAAAACcFHioVWhRu5L6tqrlpxb5D2EBAAAAOMfS7Ye18/Bpp20P9m6mkJDg6lWQCAsAAACAg2EYevN753UVmtWsqGsuru2nFvkXYQEAAAD4/77bkaYdqaectj0QpL0KkhTm7wYAAABJW7+Qdi6U7Ln+bgngfX2el2pc5O9WuDnbq/Cb07YmNWJ0Xes6fmqR/xEWAADwt9+WSl+O8ncrAN/p/JC/W+BR8q40bfvTpVehVzOFBmmvgsQwJAAA/G/P9/5uARD0DMPQm985z4DUuHqM+l9W108tMgfCAgAA/pZzquRjAHjV8t+OaEvKSadt9/dsGtS9ChLDkAAA8L8854WflNBdSujmn7YAvhAX7+8WODlbq+Dcq9CwWrQGXB7cvQoSYQEAAP/Ly3T+uWkfqfOD/mkLEIRW7D6qTX+ccNp2f8+mCgtlEA5/AwAA+JtrWIiI8U87gCDkqVahftUKGtSmnp9aZC6EBQAA/M0tLFT0TzuAILRqT7rWHzjutO3+Hk0VTq+CJMICAAD+51qzQM8C4DOutQr1KlfQDW3NVVPhT4QFAAD8jWFIgF+s3puuNfuOOW27r2cTRYTxK3Ih/iYAAPA3wgLgF2+59CrUiYvSP9rRq3AuZkMCAJhanr1AX205pP1HM0s+OBAZBXrsjPN7S1p3REe37/JTg4DgkJFr18970p223dujiSLDQv3UInMiLAAATO2puVv15cYUfzfDa6KVo8einLd9uC5NKYZ/2gMEq1qxkbq5fX1/N8N0GIYEADCt/AJDC3455O9meFW0ct22ZRlRHo4E4E33dG+iqHB6FVwRFgAAppWZZ1eevcDfzfCqaFuO27ZMERYAX2pcPUa3dmzg72aYEsOQAACmlZlrd9t27SW1LbWqar2cLOnA/34uUIj6XtpQstn81yggiNSrXEHD/9aQXoUiEBYAAKblKSy8fWsbS4UF/ZErTf/fjyGRFfX2kLb+aw8AnMNCT1sAgNVk5OY7/RwZFmKtoCCxIBsAU7PYExcAYCWuPQsVIy3YIc4aCwBMjLAAADAt17AQQ1gAAJ8iLAAATCszzzksREdYsADRLSxU9E87AMADwoKJ7d+/X2+//bZuvPFGNWvWTNHR0YqKilJ8fLwGDhyomTNnym53L/4DAKtwrVmw5jAkahYAmJcFn7rW8M9//lMvvviiDMN9Cc8///xTf/75p+bPn6/XX39dX3zxhRo0YG5gANbDMCQA8C96FkwqNTVVhmEoJiZGQ4cOVWJiolasWKH169frk08+UYcOHSRJ69atU58+fZSRkVHCFQEg8FDgDAD+RVgwqWrVqumVV15RamqqPvnkE40YMUKdO3dWu3btNHToUK1atUo333yzJOn333/X66+/7ucWA0D5y3DrWbBizYLrMCRqFgCYB2HBpF555RWNGzdOlSpV8rg/NDRUkydPVkREhCTpiy++8GXzAMAnGIYEAP5FWAhg1apV06WXXipJ2rNnj59bAwDlLzPPucA5JoKwAAC+RFgIcLm5uZLO9jQAgNUEZ88Cw5AAmAdhIYClpaVpx44dkqSWLVv6uTUAUP7cC5wt+MUIU6cCMDELfkUTPCZOnOhYZ6Gw2Pl8pKSkFLs/NTX1gtoFAOXFdZ2F4OhZICwAMA8LPnWDw5o1a/TGG29IkuLj43Xvvfee9zXq169fzq0CgPLFMCQA8C+GIQWgv/76S//4xz9kt9tls9n00UcfKTo62t/NAoByFxzrLDAMCYB5WfCp61s2m63M10hMTNSIESNKdezp06d13XXXOYYQTZgwQb169bqg+x48eLDY/ampqerYseMFXRsAyoP7OgsW+2fLMNx7FsL58geAeVjsqWttOTk5GjBggDZs2CBJeuyxxzRu3LgLvl58fHx5NQ0Ayp09v0C59gKnbTERFitwzs+TCpwDEcOQAJgJYaGMCmcjKos6deqUeIzdbtfNN9+s5ORkSdKdd96piRMnlvneAGBWrmssSBbsWXDtVZAYhgTAVCz21PW9Fi1aeP0eBQUFGjZsmBYsWCBJuuWWWzRlyhSv3xcA/Mm1XkEiLACAr1HgHABGjx6tmTNnSpL69++vTz/9VCEh/KcDYG0ew4LVhiERFgCYHL9xmtwjjzyiDz/8UJLUu3dvzZkzR2FhFvtmDQA8cC1ujgoPUVioxf7Zcg0LYRWkEIsFIgABzWJPXWt5/vnnNWnSJElSp06dNH/+fEVGRvq5VQDgG5kuC7IxbSoA+J4Fn7zW8Pbbb+uFF16QJNWrV0+vvvqq9u3bV+w5F110kcLDw33RPADwusw8i0+bKrF6MwDTs+CT1xq+/PJLx+s///xTXbp0KfGcffv2qVGjRl5sFQD4jmvNQnSEBf/JYvVmACbHMCQAgCm5r95swbH8DEMCYHIW/JrGGpYtW+bvJgCAX2W41CwwDAkAfI+eBQCAKbn2LBAWAMD3CAsAAFNynTq1oiVrFlyHIVGzAMBcCAsAAFOiZwEA/I+wAAAwpaw815oFKxY4ExYAmBthAQBgSq7DkKzZs8AwJADmRlgAAJgSw5AAwP8ICwAAU3IrcGYYEgD4HGEBAGBKmXkuPQuWnA2JFZwBmBthAQBgSpkui7JVtOQwJFZwBmBuhAUAgCkFR4Ezw5AAmBthAQBgOmfyC5RnL3DaFhxTpzIMCYC5EBYAAKaT5TIESbJgz0JBvmTPdt5GzwIAkyEsAABMJ8OluFmyYFhw7VWQpIho37cDAIpBWAAAmI7rGguSBWdD8hgWGIYEwFwICwAA03Etbq4QHqrQEJufWuMlZ7LctzEMCYDJEBYAAKYTHKs3u0ybGhImhUb4py0AUATCAgDAdNzXWAiGmZBiJJvFek8ABDzCAgDAdFx7FqKtVq8gMW0qgIBAWAAAmE6my2xIrN4MAP5BWAAAmI776s1BMgwJAEyGsAAAMJ3gKHBmGBIA8yMsAABMx73A2YphgWFIAMyPsAAAMB33YUhWDAsMQwJgfoQFAIDpZLkUOMdEULMAAP5AWAAAmE6GyzAka/YsuA5DomYBgPkQFgAAphOcBc70LAAwH8ICAMB0XMOCNQucCQsAzI+wAAAwneAscGYYEgDzISwAAEzHfRiSFQucmToVgPkRFgAAphMc6ywwDAmA+REWAACmkmcvUF5+gdO26AjCAgD4A2EBAGAqrmssSMHSs0DNAgDzISwAAEzFtbhZsmDNgmFQswAgIBAWAACm4lqvIEkxVhuGZM+RDOehVoQFAGZEWAAAmIprz0J0RKhCQmx+ao2XuA5BkqRwwgIA8yEsAABMJThWb85w30bPAgATIiwAAEwlOFZvznLfFh7t+3YAQAkICwAAU8nMc65ZiI6wWHGz5D4MKTxGCuGfZADmw5MJAGAqQTkMiSFIAEyKsAAAMBXXAmdrDkNiQTYAgYGwAAAwleDoWWBBNgCBgbAAADAV9wJnK9YsMAwJQGAgLASgxYsXy2azOf48//zz/m4SAJSbDJdF2Sy3IJvEMCQAAYOwEGAyMzN17733+rsZAOA1WXnBOAyJsADAnAgLAeaf//ynDhw4oJo1a/q7KQDgFa4FzjFBMQyJmgUA5kRYCCAbNmzQW2+9pcjISL344ov+bg4AeEVwFjjTswDAnAgLASI/P1933XWX8vPz9fTTT6tp06b+bhIAeEWmS80CU6cCgP8QFgLEpEmTtGnTJjVv3lxPPPGEv5sDAF7jNgwpKAqcGYYEwJwICwFg//79Gj9+vCTpvffeU2RkpJ9bBADekxkUBc5MnQogMBAWAsC9996rrKws3XbbberVq5e/mwMAXuW+zoIVwwLDkAAEBgs+ga1lxowZWrJkiSpXrqzXX3+9XK+dkpJS7P7U1NRyvR8AlCTPXqAz+YbTtmhLzoZEWAAQGAgLJnbs2DE9/PDDkqSXX3653KdLrV+/frleDwDKyrVXQQqWngVqFgCYE8OQTOyxxx5TWlqarrjiCt19993+bg4AeJ1rcbNEzQIA+JMFn8C+ZbPZynyNxMREjRgxwmnbsmXLlJiYqNDQUL3//vsKCSn/XHfw4MFi96empqpjx47lfl8AKIprcbMkRYczDAkA/IWwYEK5ubkaPXq0JOnBBx/U5Zdf7pX7xMfHe+W6AHCh3BZkiwhVSEjZv5QxlfwzUn6u8zaGIQEwKcJCGe3YsaPM16hTp47Tz3PnztVvv/2m8PBwtWrVSjNnznQ7Z/v27Y7X27ZtcxxzxRVXKCEhocxtAgB/yHBZkM2aQ5Ay3bfRswDApCz4FPatFi1alPs1c3PPfuN05swZ3XXXXSUe/+WXX+rLL7+UdHZIE2EBQKAKymlTJSki2vftAIBSoMAZAGAarmEhKKZNlaRwehYAmJPfvrI5deqUTp8+rfz8/BKPbdCggQ9aZB4jRoxwK3h2tWzZMvXs2VOSNH78eD3//PPebxgAeJl7zYIVexZcZkIKjZDCIvzTFgAogU+fwt9++60mT56sFStW6NixY6U6x2azyW53nx0DAGA9mXnOXyBZchjSmSznn6lXAGBiPnsKP/jgg3r33XclSYZhlHA0ACAYua6zEBQFzsyEBMDEfPIUnjFjht555x1JUlRUlAYOHKh27dqpatWqXlk/AAAQmNyGIVkyLLAgG4DA4ZOn8JQpUyRJ9evX1w8//KAmTZr44rYAgADj2rNQMRgKnAkLAEzMJ2Hhl19+kc1m0/jx4wkK5aRHjx4M5wJgOVnBuM4CYQGAiflkDNCZM2ckSW3atPHF7QAAASozLwhnQ6JmAYCJ+SQsNGrUSJKUkZFR/IEAgKAWnAXO9CwAMC+fhIUbbrhBkvT999/74nYAgADlXuBMzQIA+JNPwsKjjz6qBg0a6I033tDOnTt9cUsAQADKzA2CdRaYOhVAAPFJWIiLi9M333yjWrVqqVOnTpo8ebKOHz/ui1sDAAJIcAxDYupUAIHDJ0/hxo0bS5KysrJ04sQJPfDAA3rwwQdVvXp1RUdHF3uuzWbTnj17fNFMAIAfGYbhNgwpOHoWCAsAzMsnT+H9+/c7/WwYhgzDUFpaWonn2mw2L7UKAGAmefkFshc4TwkdHUHNAgD4k0/Cwu233+6L2wAAAphrvYJk1Z4Fpk4FEDh88hROTEz0xW0AAAHMdQiSZNWaBXoWAAQOnxQ4AwBQEtfiZpuNYUgA4G+EBQCAKbitsRARZs26NaZOBRBA/NK/m52drQ0bNujw4cPKysrSwIEDFRsb64+mAABMwn3aVAv2KhQU0LMAIKD4NCwcPHhQTz/9tObMmaMzZ844trdv316tWrVy/Dxt2jRNmTJFcXFxWrp0qTW/WQIAOHEtcLZkvYI9W5LzjE+EBQBm5rNhSGvWrFGbNm00Y8YM5eXlOaZP9aR///765Zdf9MMPP2jp0qW+aiIAwI8y89yHIVmOa6+CxDAkAKbmk7Bw4sQJDRgwQMeOHVPt2rU1efJkbd26tcjja9asqWuvvVaStGjRIl80EQDgZ241C1YchuQ6baokhRe/OCkA+JNPvrZ56623lJaWpurVq2vVqlVq0KBBief06dNH8+fP19q1a33QQgCAvwXl6s2ySeEV/NIUACgNn/QsLFiwQDabTY888kipgoIkXXzxxZKkPXv2eLNpAACTyAiGmgVPMyFRlwfAxHwSFnbv3i1J6tatW6nPqVKliiTp1KlTXmkTAMBc3IchBUNYoLgZgLn5JCzk5ORIksLDw0t9Tmbm2QdqhQp0zwJAMAjKYUiEBQAm55OwULNmTUnSvn37Sn3O5s2bJUl169b1RpMAACYTlLMhERYAmJxPwsIVV1whSVq8eHGpjjcMQ1OnTpXNZlPXrl292TQAgEm4r7MQBLMhMW0qAJPzSVi47bbbZBiGPvvsM0ePQXEeffRRbdmyRZJ0++23e7l1AAAzcF/BmZ4FAPA3n4SFAQMGqGfPnrLb7erdu7fee+89paWlOfbb7XYdOnRIc+bMUdeuXfXmm2/KZrPphhtuUKdOnXzRRACAn1HgDADm47Mn8ZdffqnevXtr06ZNGjNmjMaMGSPb/58urk2bNk7HGoahK6+8UklJSb5qHgDAz9wLnK04DMnD1KkAYGI+6VmQpMqVK2vVqlV66qmnFBsbK8MwPP6pUKGCxo0bp2XLlikmhm9cACBYuA1DsmSBs2vNAv/OATA3nz6JIyIi9OKLL+rpp5/W8uXLtX79eqWlpSk/P1/VqlVTmzZt1KdPH8XFxfmyWQAAPzMMQ5l5wbgoG2EBgLn55UkcExOjfv36qV+/fv64PQDAZHLtBcovMJy2ERYAwP98NgwJAICiuNYrSEydCgBm4JevbbKzs7VhwwYdPnxYWVlZGjhwoGJjY/3RFACACbiusSCxgjMAmIFPn8QHDx7U008/rTlz5ujMmTOO7e3bt1erVq0cP0+bNk1TpkxRXFycli5d6pg1CQBgTa7FzSE2qUK4FXsWCAsAAovPhiGtWbNGbdq00YwZM5SXl+eY/ciT/v3765dfftEPP/ygpUuX+qqJAAA/ycxznwnJkl8UMXUqgADjk7Bw4sQJDRgwQMeOHVPt2rU1efJkbd26tcjja9asqWuvvVaStGjRIl80EQDgR0GxerPE1KkAAo5PnsZvvfWW0tLSVL16da1atUoNGjQo8Zw+ffpo/vz5Wrt2rQ9aCADwJ/fVmy04BEliGBKAgOOTnoUFCxbIZrPpkUceKVVQkKSLL75YkrRnzx5vNg0AYAJZuUGwxoI9Tyo447yNYUgATM4nYWH37t2SpG7dupX6nCpVqkiSTp065ZU2AQDMIyhXb5boWQBgej4JCzk5OZKk8PDwUp+TmXm2q7ZChQpeaRMAwDzchyFZMSxkum8jLAAwOZ+EhZo1a0qS9u3bV+pzNm/eLEmqW7euN5oEADCRDJfZkCpasWaBsAAgAPkkLFxxxRWSpMWLF5fqeMMwNHXqVNlsNnXt2tWbTQMAmEBQ9iyERUkhFgxFACzFJ2Hhtttuk2EY+uyzzxw9BsV59NFHtWXLFknS7bff7uXWAQD8zXUFZ2uu3sy0qQACj0/CwoABA9SzZ0/Z7Xb17t1b7733ntLS0hz77Xa7Dh06pDlz5qhr16568803ZbPZdMMNN6hTp06+aCIAwI+ComfhTJbzz4QFAAHAZ0/jL7/8Ur1799amTZs0ZswYjRkzxrE6Z5s2bZyONQxDV155pZKSknzVPACAH7mu4BwdYcHhOazeDCAA+aRnQZIqV66sVatW6amnnlJsbKwMw/D4p0KFCho3bpyWLVummBi+dQGAYJDBMCQAMCWfPo0jIiL04osv6umnn9by5cu1fv16paWlKT8/X9WqVVObNm3Up08fxcXF+bJZASEzM1NJSUmaO3eudu7cqaNHj6py5cqqV6+eOnfurP79+6tv377+biYAXJCgGIbE6s0AApBfnsYxMTHq16+f+vXr54/bB5zk5GSNHDlSBw4ccNqelpamtLQ0bdq0ST/99BNhAUDAcg0L1uxZYBgSgMBjwaextXz33Xfq37+/cnJyVLlyZd1zzz3q0aOHatasqaysLO3YsUMLFy7UX3/95e+mAsAFc1vB2ZJhgWFIAAKPaZ7Gf/31lxYuXKijR48qISFBf//73xUdHe3vZvnVkSNHNHjwYOXk5Ojyyy/XkiVLVKtWLadjOnfurDvvvFN5eXl+aiUAlI1hGB6GIQVDgTNhAYD5+SQs7NixQ+PHj5fNZtOUKVNUuXJlp/1fffWVhgwZouzsbMe2+Ph4zZ8/X5dffrkvmmhKTz31lNLT0xUdHa158+a5BYVzRURE+LBlAFB+cu0FKjCct8VEmOa7rPJDWAAQgHwyG9K8efP0xRdf6NChQ25BIS0tTUOHDlVWVpbTrEgHDx5U//79lZGR4fmiFnf8+HHNmDFDkjR06FA1bNjQzy0CAO9wHYIkBcswJGoWAJifT8LC999/L5vNpr///e9u+yZPnqyMjAyFhYXp9ddf15YtW/Tqq68qJCREhw4d0tSpU33RRNNZuHCho6fl+uuvd2zPysrS7t27dfjwYRmGUdTpABAwXIcgScFS4EzPAgDz80lY+OOPPyS5L74mnV2szWazafjw4Ro7dqxat26txx57TKNGjZJhGPrqq6980UTTWb16teN169attW7dOvXt21eVKlVSs2bNVKdOHdWqVUtjxoyhuBlAQHPtWQixSVHhPlsGyHcICwACkE++uklLS5Mk1axZ02n70aNH9euvv8pms2nIkCFO+66//npNnTpV27dv90UTTefc952cnKw777xTdrvzP6hHjhzRu+++qy+//FJLlizRZZdddl73SElJKXZ/amrqeV0PAC5EpsuCbDGRYbLZbH5qjRcxdSqAAOSTsFA4nCYnJ8dp+4oVKySdLc7t0qWL0746depIkk6cOOH9BprQsWPHHK/vuece2Ww2/d///Z+GDx+uWrVqaffu3Zo4caKSkpJ0+PBhDRw4UFu2bFFsbGyp71G/fn1vNB0AzktQrLEgMXUqgIDkk37eqlWrSvrfcKRC33//vSSpffv2brP5FH6LXrFicH7zkpn5v2+gcnJyNG3aND3zzDOqX7++IiIi1KpVKyUmJuruu++WJO3fv1/vvfeev5oLABcsKNZYkBiGBCAg+SQsFA6PKZzdRzrb2zBnzhzZbDb16tXL7ZzC1YqLmy7UDGw2W5n/JCUluV03KirK8frSSy/VsGHDPN7/pZdeUmRkpCRp1qxZ59X2gwcPFvtn7dq153U9ALgQWXkuYSHCgmssSAxDAhCQfPL1zeDBg7V06VItWLBAgwcPVpcuXTRr1iylpaUpJCREt956q9s5a9askaSgnTK0UqVKjtd9+/Yt8rhq1aqpffv2WrlypbZs2aK8vLxSr7kQHx9f5nYCQFlleKhZsJyCfOlMlvM2ehYABACfPJGHDx+u6dOna8WKFZozZ47mzJnj2Ddy5Ei1aNHC7Zy5c+fKZrOpU6dOvmjiBduxY0eZr1FYn3Gu+vXrO2ZEKqm2oHB/QUGBjh07ptq1a5e5TQDgK+6rN1swLLgGBYmwACAg+OSJHBISosWLF2v8+PGaM2eODh8+rDp16uj222/XP//5T7fjFy5cqP3798tms6lfv36+aOIF8xR0ysPFF1/sCFX5+fnFHnvu/rAwC/4jC8DSgqLA2XUIksQwJAABwWdP5JiYGP3nP//Rf/7znxKP7dy5s/bt2ycpeIchdevWzfF67969xR67Z88eSWfrHAqLyQEgULgXOFuwZsFjWKBnAYD5mXLVmypVqqhhw4ZBGxSks2GhRo0akqQFCxYU2buwb98+bd68WdLZkBUSYsr/pABQpKAYhuQ6bWpImBRauvoyAPAnfrM0qdDQUD322GOSzs4M9e9//9vtGLvdrvvuu08FBQWSzq7HAACBJjPP+cuQihFWDAsuPQvhMZIVF54DYDmEBRN78MEH1bZtW0nSCy+8oFtvvVVLlizRxo0bNWfOHHXr1k1LliyRJPXr10833nijP5sLABfEtWch2pI9C6yxACAwWfCJbB1RUVFauHCh+vfvrw0bNmjmzJmaOXOm23H9+vXTzJkzZeNbKgAByL3AOQhqFggLAAIEPQsmV6dOHa1evVrvv/++unfvrho1aig8PFy1a9fW9ddfr7lz52rRokVO6zIAQCAJihWcCQsAApQFn8jWExYWptGjR2v06NH+bgoAlLvMYFiUjdWbAQQoehYAAH4VHOssuMyGRM8CgABBWAAA+I1hGMrMcxmGFAyzIREWAAQIwgIAwG9yzhSowHDeFhSLshEWAAQIwgIAwG9ci5slq9YsuA5DomYBQGDwSVhISEhQkyZNtHv37lKf88cff6hx48Zq0qSJF1sGAPAn13oFyao1C/QsAAhMPnkiHzhwQDabTXl5eaU+58yZM9q/fz9rBwCAhbn2LISG2BQZZsFOb8ICgABlwScyACBQuPYsxESEWvNLIqZOBRCgTBsWTp48KUmKjo72c0sAAN7iOhOSJYcgSUydCiBgmTYsfPrpp5Kkhg0b+rklAABvyQiGBdkkhiEBCFheeSr36tXL4/aRI0cqJqb4B2Rubq727t2rtLQ02Ww29e3b1xtNBACYQJbLMKTooAkLDEMCEBi88lRetmyZbDabDON/k2cbhqF169ad13UaN26sp556qrybBwAwCdcC54pWXGNBomcBQMDySljo1q2bU4Ha8uXLZbPZ1K5du2J7Fmw2m6KiolSnTh116tRJgwcPLrEnAgAQuDJdhyFZcfVmw6BmAUDA8lrPwrlCQs6WRiQlJalVq1beuCUAIAAFRYGzPVcynEMRYQFAoPDJU3n48OGy2WyqUqWKL24HAAgQrsOQLFng7DoESaJmAUDA8MlTOSkpyRe3AQAEGLd1FiwZFjLct9GzACBAmOqpvGfPHh09elSNGjVSrVq1/N0cAICXuYYFSxY4e+pZCGcNIQCBwSfrLKSlpWny5MmaPHmyY7G1c+3evVvt2rVT8+bN1alTJ9WrV0833nijjh8/7ovmAQD8xLXAOdqKBc6uYSE8Rgox7TJHAODEJ0+ruXPnasyYMXrzzTcVFxfntC83N1fXXnutNm/eLMMwZBiGCgoKNG/ePA0YMMAXzQMA+ElQFDi7zYRErwKAwOGTsLB06VLZbDYNGjTIbV9SUpL27NkjSbr++uv15ptvqn///jIMQytXrtSsWbN80UQAgB8EZYEz9QoAAohPwsKuXbskSVdeeaXbvhkzZkg6u+rzvHnz9MADD2j+/Pnq06ePDMPQzJkzfdFEAIAfuBc4B0HNAjMhAQggPgkLR44ckSTFx8c7bc/Oztbq1atls9l09913O+274447JEkbN270RRMBAH7gWrNgyWFIZ+hZABC4fBIWTpw4cfZmLgVdq1ev1pkzZ2Sz2dSnTx+nfQkJCZLOFkcDAKzHMAy3mgWGIQGAufgkLFSseLbL9fDhw07bC1d6btWqlduCbeHh4ZKksDAL/sMBAFD2mXwZhvM2S/YsEBYABDCfhIUWLVpIkpYsWeK0/csvv5TNZlP37t3dzikMFqy3AADW5FrcLEnREVasWXCdDYmaBQCBwydf4Vx33XVavXq1PvjgA7Vs2VJdu3ZVUlKStm/fLpvNphtuuMHtnMJahXr16vmiiQAAH3OtV5AYhgQAZuOTp/KYMWM0efJkpaamasyYMU77/va3v6lnz55u5yxYsEA2m00dOnTwRRMBAD7mOhNSWIhNkWEWXKyMsAAggPnkqRwXF6fvvvtObdu2dSy8ZhiGunbtqtmzZ7sdv2XLFq1bt06SdNVVV/miiQAAH/O0xoLNZvNTa7yIqVMBBDCf9fe2bNlS69ev1759+3T48GHVqVNHjRo1KvL4xMRESWfXXwAAWI9rz4Ili5slDzUL9CwACBw+fzInJCQ4pkUtymWXXabLLrvMRy0CAPiDe8+CBYubJYYhAQhoFhwcCgAIBFl5zgXO0RFW7VlgGBKAwOXzJ3NBQYGSk5O1atUqHT58WFlZWXrxxRdVp04dxzF5eXmy2+0KDQ1VZGSkr5sIAPABhiEBgPn59Mm8cOFCPfjggzpw4IDT9scee8wpLHz44Yd64IEHVLFiRR06dEgxMTxYAcBqGIYEAObns2FIU6dO1YABA7R//34ZhqFq1arJcF268/+78847FRcXp4yMDP33v//1VRMBAD7k2rNgyTUWJMICgIDmk7Dw+++/6/7775d0dnaj7du3Ky0trcjjIyIidOONN8owDC1dutQXTQQA+FiGy6JslhyGlG+X7DnO26hZABBAfBIWJk2aJLvdrosvvlhff/21WrRoUeI5Xbt2lSRt2rTJ280DAPhBUPQsnMl030bPAoAA4pOw8MMPP8hms2ns2LGKiIgo1TlNmzaVJB08eNCbTQMA+ElQFDi7DkGSCAsAAopPwkJKSookndfaCYVFzVlZWV5pEwDAvzLznMNCdIQFC5w9hYVwwgKAwOGTsGCz2SSd3y/+6enpkqS4uDivtAkA4F+ZLjULlhyG5DptamiEFFa6HnYAMAOfhIV69epJkvbu3Vvqc1asWCFJaty4sVfaBADwr6AchsQQJAABxidhoUePHjIMQx999FGpjj958qTef/992Ww29erVy8utAwD4g/s6C0EQFhiCBCDA+CQsjB49WjabTcuXL1dSUlKxx6anp2vgwIE6fPiwwsLCdM899/iiiQAAH3PvWbBizQKrNwMIbD4JC23atNFDDz0kwzA0atQo3XLLLZo9e7Zj/88//6wZM2bo/vvvV9OmTfXjjz/KZrPpn//8pxo2bOiLJgIAfKigwFBmXjDULDAMCUBg89mT+bXXXlNubq7ee+89ffHFF/riiy8chc+jR492HFe4qvPYsWP17LPP+qp5prZkyRIlJSVp7dq1Onz4sAoKClSjRg21bdtWQ4YM0U033aSQEJ8txg0AZZZ9Jt9tW0yEFcOCy8QehAUAAcZnv2HabDa9++67+uabb9SjRw/ZbDYZhuH0R5L+9re/adGiRXr99dd91TTTys3N1T/+8Q9de+21mjVrlvbt26fs7Gzl5uYqJSVFX331lQYPHqwePXroxIkT/m4uAJSa6xAkyao9C67DkFi9GUBg8fmT+aqrrtJVV12l06dPa9OmTUpLS1N+fr6qVaumyy+/XNWrV/d1k0zrwQcf1JdffilJqlmzpsaNG6e2bdsqPDxcW7du1SuvvKIDBw7op59+0uDBg7VkyRI/txgASse1uFmSYixZs8AwJACBzW9f41SqVEndunXz1+1N76+//tKHH34oSapSpYo2bNig+Ph4x/4uXbrotttu02WXXab9+/frm2++0fr169W+fXt/NRkASs11jYXwUJsiwwgLAGA2DHQ3qTVr1qigoECSNHLkSKegUCg2NlYPP/yw4+dVq1b5rH0AUBZBMW2q5CEsMAwJQGAxTVg4fvy4jhw54qhdCHZ5eXmO18UtTNekSROP5wCAmbnWLFiyuFli6lQAAc+rYcFut2vbtm3asGGDjhw54rY/JydHzz33nOLj41W9enXVrl1blSpV0j/+8Q/9+uuv3mya6V100UWO18WtfL1nzx6P5wCAmWXmBcHqzRLDkAAEPK+EBcMw9Nxzz6l69eq67LLL1LFjR9WuXVtdunTRunXrJJ39Fvzqq6/Wiy++qNTUVMeMSFlZWfrvf/+rjh076vvvv/dG8wJC69at1alTJ0lSUlKSDh065HbM6dOn9cYbb0g62/vQt29fXzYRAC6Ya81CtBWLmyXCAoCA55WvckaOHKlPPvlEkpyGFf3888+65pprtGbNGk2ePFk//fSTJKlq1apq1qyZ7Ha7tm/fruzsbGVnZ+u2227Trl27FBcX541mml5iYqKuueYa7du3T23btnXMhhQWFqZt27bp1Vdf1b59+1S9enV99tlnioiIOK/rp6SkFLs/NTW1LM0HgCK5r95s1Z4Fpk4FENjK/emcnJysjz/+WDabTZGRkbruuuuUkJCgAwcOaOHChTpx4oQmTZqkzz//XOHh4Xr33Xc1atQoxwJt2dnZGj9+vP7zn//oyJEjSkpK0kMPPVTezQwIzZs317p16/Tee+/plVde0aOPPuq0Pzw8XI899pgeeughjwXQJalfv355NRUAzotbgbNlaxboWQAQ2Mr96ZyYmCjp7LoAP/zwg1q2bOnYt3PnTvXq1UsffPCBCgoK9Pjjj+vOO+90Or9ChQp69dVXtXXrVn3zzTdatGhR0IYFSVqwYIE+++wzZWRkuO07c+aMZs+erRo1aujxxx93BC4AMDu3AmfL9iwQFgAEtnKvWVizZo1sNpsefvhhp6AgSS1atNDDDz+s/PyzY1WHDRtW5HVuv/12STJ9obPNZivzn6SkJI/XfvTRRzVy5Ejt3LlTAwcO1MqVK5WRkaHs7Gxt3LhRI0eO1B9//KEnnnhC//jHPxx/r6V18ODBYv+sXbu2HP6GAMCde4FzsNQsMAwJQGAp969yCgtx//a3v3ncf+72pk2bFnmdZs2aSZKOHTtWjq0LHIsWLdLrr78uSRoxYoSjx6ZQmzZtNH36dMXHx+vf//635s6dq8mTJ+uBBx4o9T0uZOgSAJSHDJcCZ0v2LBgGU6cCCHjl/nTOzMyUzWZT1apVPe6vXLmy43VkZGSR14mKipJk/rUDduzYUeZr1KlTx21b4erNNptN//d//1fkuU8//bQmTZqkjIwMTZ8+/bzCAgD4S1AMQzqTLcll7SDCAoAA47Wnc1Hj5602rr5FixZeuW5hCKlZs6bq1atX5HFRUVG6+OKLtWbNGu3cudMrbQGA8ua+KJsFhyG5DkGSGIYEIOCYZgVnOAsLO5vj7HZ7CUeeLXQ+9xwAMDvXmgVL9iy4DkGS6FkAEHAICyaVkJAgSUpPTy92qNOxY8e0bds2p3MAwOxcF2Wz5DoLbj0LNim8gl+aAgAXymtP58mTJ6tmzZpu29PS0hyv//WvfxV5/rnHBaP+/ftr4cKFkqSxY8dqwYIFbouuFRQU6MEHH3TUdfz973/3eTsB4EK4rbMQDGEhoqJksaG4AKzPa0/n9957r8h9hXULL7zwgrduH/BGjBihN954Qzt27NDSpUvVvn17PfDAA7rssssUGhqq7du367333tOqVaskSbVq1dIjjzzi51YDQOkERYEzMyEBsACvPJ0Nwyj5IBQrIiJCixcv1oABA7RlyxZt3bpVd999t8djExISNHfuXFWvXt3HrQSA81dQYCgrLwiHIUVE+6cdAFAG5f50Tk5OLu9LBq2GDRtq3bp1mjlzpr744gtt3LhRR44ckWEYqlq1qi699FINHDhQw4cPV0wM31gBCAxZZ9wXkIyx4qJsrN4MwALKPSx07969vC8Z1MLDwzVs2LBiV7sGgEDiOgRJkmIirNiz4DoMiWlTAQQeZkMCAPiUa3GzZNGahTNZzj/TswAgABEWAAA+5dqzEBEaoogwC/5zxDAkABZgwaczAMDM3KdNtWC9guR56lQACDCEBQCAT7kuyGbJIUgSU6cCsATCAgDAp1yHIVly2lSJYUgALIGwAADwqcw857AQHREsw5AICwACD2EBAOBTQbF6s8TUqQAsgbAAAPCpjNwgWL1ZomcBgCUQFgAAPhU8PQuEBQCBj7AAAPCp4C1wZhgSgMBDWAAA+FTwrLPA1KkAAh9hAQDgUwxDAoDAQVgAAPhUZp7LomwRFgwL9jwpP895G8OQAAQgwgIAwKeComfhTKb7NnoWAAQgwgIAwKfcC5wtWLPgOgRJIiwACEiEBQCAT7mus2DJngVPYSGcsAAg8BAWAAA+FRTDkFxnQgqLkkIt+D4BWB5hAQDgM/kFhrLPBMEKzsyEBMAiCAsAAJ/JyrO7bbNmzwJhAYA1EBYAAD6T6VKvIEkxEUFQ4Ey9AoAARVgAAPiM6+rNklV7Fli9GYA1EBYAAD7jWtwcERai8FAL/lPEMCQAFmHBJzQAwKzc11iwYK+C5CEssHozgMBEWAAA+IzrMKQYKy7IJtGzAMAyCAsAAJ/JdJkNKSYiWHoWCAsAAhNhAQDgM66zIVmyuFkiLACwDMICAMBngmL1ZsnDbEjULAAITIQFAIDPuBc4U7MAAGZGWAAA+EyG6zAkahYAwNQICwAAnwmeYUhMnQrAGggLAACfycgLlnUWWMEZgDUQFgAAPhO8PQuEBQCBibAAAPCZLLepU4OlwJlhSAACE2EBAOAzbis4W7HAuaBAOkPPAgBrICwAAHzGbQVnKw5DOpPlvo2wACBAERYAAD7jvs6CBcOC6xAkiWFIAAIWYQEA4DNuw5CsWLPgOhOSRM8CgIBFWAAA+IQ9v0A5ZwqctgVFz4ItVAqL9E9bAKCMCAsAAJ/IOpPvts2SNQueZkKy2fzTFgAoI8ICAMAnXOsVJIvOhsQaCwAshLAAAPAJj2EhGGoWCAsAAhhhAQDgExkuC7JFhoUoLNSC/wzRswDAQiz4lPa/jIwM/fjjj/rPf/6jm2++WQkJCbLZbLLZbGrUqNF5X2/btm0aPXq0mjRpogoVKqhGjRrq2rWr3n//fdnt7t/UAYAZBcW0qRJhAYClWPRJ7V/9+/fXsmXLyuVaU6dO1ZgxY5SXl+fYlpOToxUrVmjFihVKTEzUokWLVL169XK5HwB4i/u0qRb9J4hhSAAshJ4FLzAMw/G6atWq6tu3rypWPP8Feb7++mvdc889ysvLU61atfTWW29pzZo1Wrx4sW644QZJ0tq1azVo0CDl57vPMgIAZuLas2DdsEDPAgDrsOiT2r+GDBmi0aNHq0OHDmratKkkqVGjRsrI8LBQTxHOnDmjBx54QAUFBYqNjdXKlSvVpEkTx/5rrrlG999/vyZPnqwVK1bok08+0YgRI8r7rQBAucnMc/5SIybCgsXNknQmy/lnwgKAAEbPghfcfffduvXWWx1B4UL897//1d69eyVJTz31lFNQKDRx4kRVqVLF8RoAzCx4ehZchyGdf88yAJgFYcGk5s2b53hdVI9BdHS0br75ZknS9u3b9dtvv/mgZQBwYShwBoDAQ1gwqRUrVkiSLrroItWuXbvI47p37+54vXLlSq+3CwAulHuBs0WHIREWAFgIYcGEMjIydPDgQUlSixYtij323P07duzwarsAoCyCZxiSa1hgGBKAwGXRJ3VgS0lJcbyOj48v9tj69es7XhcGjAu5jyepqanndT0AKE6my6Js1h2GxNSpAKzDok/qwHb69GnH65KmXI2J+d8/Qucz25LkHDQAwNuCZ50FhiEBsA6GIZlQTk6O43VERESxx0ZGRjpeZ2dne61NAFBWWXkuYcGqU6cyDAmAhVj0a52S2Wy2Ml8jMTHRK2sbREVFOV6fu3KzJ7m5uY7XFSpUOK/7lDRsKTU1VR07djyvawJAUTJchiFZt2eBYUgArMOiT+rAVqlSJcfrkoYWZWb+7xus810luqR6CAAoT0FR4GwYDEMCYCkWfFKXTnnMHFSnTp1yaIm7evXqOV6XVIR8bu8ANQgAzCwo1lnIz5MKnN8nw5AABDILPqlLp6QpSf2pUqVKql+/vg4ePKidO3cWe+y5+1u2bOntpgHABQuKAmfXXgWJngUAAY0CZ5Pq0qWLJGnXrl06fPhwkcctX77c8bpz585ebxcAXAh7foFy7QVO2ypacVE213oFibAAIKARFkxq4MCBjtdJSUkej8nKytLs2bMlSa1atVLz5s190DIAOH+uayxIQdSzEB7t+3YAQDkhLJjUoEGD1LhxY0nSyy+/rD179rgd8/jjj+v48eOO1wBgVpku06ZKUnREEISF8GgpxII9KACChgWf1P63e/durVixwmlb4axGGRkZbj0F11xzjWrXru20LTw8XG+//bb69++vU6dOqXPnznr22WfVsWNHHT9+XFOnTtWXX34p6eyQpWHDhnnvDQFAGbkWN0sWXWeBaVMBWAxhwQtWrFihkSNHetyXnp7uti85OdktLEhSv3799P7772vMmDH666+/9MADD7gd07FjR/33v/9VaKgF/9EFYBmuxc1R4SEKC7Vg5zbTpgKwGAs+qa3lrrvu0oYNG3TXXXepcePGioqKUrVq1dSlSxe99957WrlypapXr+7vZgJAsVxrFiw5barE6s0ALMeiT2v/GjFiRLmu7HzJJZfogw8+KLfrAYCvBcW0qRLDkBAwsrOzderUKWVmZio/330CAphHaGioYmJiFBsbqwoVKvj8/hZ9WgMAzMRt9WYrFjdLngucAZM5efKkDh065O9moJTsdrtyc3N17Ngx1a1bV3FxcT69v0Wf1gAAM8nKc+1ZsGidFTULMLns7Gy3oBAWxq+DZma3/+/5eejQIUVGRioqKspn9+fTAQDwugyXmoXgGYZEzQLM5dSpU47XsbGxql27NpOkmFx+fr4OHz7s+G938uRJn4YFCpwBAF7nNgzJsmEhy/lnehZgMpmZ/+v9IigEhtDQUKdZM8/9b+gLhAUAgNe5FjhXDJaaBcICTKawmDksLIygEEBCQ0Mdw8V8XZBOWAAAeF3w9CwwDAmAtRAWAABel+lS4FyRAmcACAiEBQCA1wVPgTNhAYC1EBYAAF6X5TIMKTpowgLDkAAENsICAMDr3AqcLTsMiRWcgUByxx13yGazqVq1asrNzS322M2bN+uee+5Rq1atFBsbq4iICNWuXVtXXXWVXnvtNR05csTtHJvN5vQnLCxMderU0cCBA/Xjjz96622VK4t+tQMAMBPXmoWgWcGZsACY1unTpzV79mzZbDYdO3ZM8+bN0y233OJ2XEFBgcaNG6fXXntNoaGh6tatm/r27auYmBilpaVp1apVeuyxxzR+/Hjt2rVL9erVczq/WrVqGjNmjCQpJydHmzdv1vz58/XVV19p1qxZuummm3zyfi+URZ/WAAAzyXSpWajIMCQAfjZr1ixlZmbqkUce0RtvvKFp06Z5DAvPPPOMXnvtNbVt21azZs1S06ZN3Y7ZuHGjnnjiCWVnZ7vtq169up5//nmnbR9++KHuuusujRs3zvRhgWFIAACvcx2GZMkC54J8ye7yiwI9C4BpTZs2TWFhYRo3bpx69uyp77//XgcOHHA65rffftPEiRNVo0YNLVmyxGNQkKS2bdvq22+/VaNGjUp17zvuuEMxMTHav3+/x+FLZmLBpzUAwEzO5Bcoz17gtM2SYcG1V0EiLCBgFBQYOp6V5+9mlFqV6AiFhNgu+Pzt27dr9erV6tevn2rVqqXhw4fr+++/V2JiolMvwEcffaT8/HyNHj1aNWrUKPG6hQunnQ+b7cLfhy9Y8GkNADAT1wXZJIsOQyIsIIAdz8pTu//7zt/NKLUNz/ZRtYqRF3z+tGnTJEnDhg2TJN1www267777lJiYqOeee04hIWcH36xatUqS1LNnzzK22NlHH32kzMxMJSQkqHr16uV67fJmwac1AMBMMvPy3bZFW3E2JI9hgZoFwGzOnDmjTz75RLGxsRo4cKAkqWLFiho0aJA+/fRTfffdd+rbt68k6fDhw5KkunXrul1n2bJlWrZsmdO2Hj16qEePHk7bjh496uityMnJ0ZYtW7RkyRKFhIRo4sSJ5frevIGwAADwKk89C5acDcl12tSQcCkswj9tAVCk+fPn68iRIxo1apSioqIc24cPH65PP/1U06ZNc4SF4ixbtkwvvPCC23bXsJCenu44LjQ0VNWrV9eAAQP06KOPqmvXrmV7Mz5AgTMAwKtci5srhIcqtAxjjU2LaVOBgFA4BGn48OFO23v37q169epp/vz5OnbsmCSpVq1akqRDhw65Xef555+XYRgyDEOff/55kfe76KKLHMfZ7XYdPnxY8+bNC4igINGzAADwMteeBUsWN0tMm4qAViU6Qhue7ePvZpRalegL67U7ePCgli5dKknq3r17kcd9+umnevDBB9WpUyctW7ZMycnJ6tWr1wXdM9BZ9IkNADAL17DA6s2A+YSE2MpUMBwokpKSVFBQoC5duuiiiy5y22+32/XRRx9p2rRpevDBB3X77bdrwoQJ+uCDD/TQQw+ZvhjZGwgLAACvynBZkC14ehYIC4CZGIahxMRE2Ww2ffTRR2rcuLHH43777TetWrVK69evV/v27TVu3DhNmDBB1157rT7//HOPay2cOHHCy633H4s+sQEAZpGVF6zDkAgLgJn88MMP2rdvn7p3715kUJCkkSNHatWqVZo2bZrat2+vF198UXl5eXr99dfVokULdevWTZdddpmio6OVlpamX375RWvXrlXFihV1+eWX++4N+QgFzgAAr3JbvTmCYUgAfK+wsHnEiBHFHnfLLbeoQoUK+vzzz5Wdna2QkBC99tpr2rhxo0aNGqXU1FR9+OGHmjhxohYsWKCKFStq4sSJ2rNnj2MqViux6Nc7AACzCN4CZ8ICYCYzZszQjBkzSjwuNjZWWVlZbtvbtGmjKVOmnNc9DcM4r+PNiJ4FAIBXZbrULFhy9WaJsADAkggLAACvchuGZNWwcIapUwFYD2EBAOBVDEMCgMBFWAAAeJVrz4J111kgLACwHsICAMCrsvKcaxaiI4KlZ4FhSAACH2EBAOBV7is4WzUsMHUqAOshLAAAvCpoCpwZhgTAgggLAACvci9wpmYBAAIFYQEA4FXBu84CNQsAAh9hAQDgNXn2AuXlFzhts+QwJMOgZgGAJREWAABe4zoESbJoz4I9RzKcQxFhAYAVEBYAAF6TmeceFqIjLFiz4DoESWIYEgBLICwAALzGtV5BkmKsuM6C6xAkiZ4FAJZAWAAAeI3rtKnREaEKCbH5qTVe5NazYJPCKvilKQBQnggLAACvcZ821YK9CpLnaVND+CcWMKP9+/fLZrPJZrOpdu3astvdh0tK0o4dOxzHNWrUyLE9KSnJsb2oPyNGjHC6VmZmpl566SW1bdtWFStWVGRkpOLj49W1a1c99dRT2rNnjxffcdlY9KmNYJV2Kkcz1x3Uscw8fzcFgKQ/jmU5/WzJ4maJmZCAABQWFqa//vpLX3/9ta6//nq3/dOmTVNIMaG/d+/e6tKli8d9l19+ueP16dOn1aVLF/3yyy9q2rSphg4dqmrVquno0aNau3atJkyYoCZNmqhJkyZlfk/eYNGnNoLRmr3puu+zjUonKACmxYJsAMyiU6dO2rJli6ZPn+4WFux2uz799FP16dNHy5cv93h+nz599OSTT5Z4nzfeeEO//PKL7rzzTn3wwQey2ZyHYu7bt0+5ubkX/ka8jD5SBDzDMPTxqv267cM1BAXA5OIqhPu7Cd5BWAACToUKFTR48GAtWrRIaWlpTvsWLlyov/76S3fccUeZ77Nq1SpJ0v333+8WFCQpISFBLVq0KPN9vIWeBS/IyMjQxo0btXbtWq1du1br1q3T/v37JUkNGzZ0vC5OQUGBVqxYoSVLlujnn3/Wzp07dezYMUVFRalBgwbq1q2b7rnnHl166aXefTMml2vP13PzftWs9Qf93RQApXD1xbX93QTvcBuGxLSpCDAFBVL2MX+3ovQqVC2XuqA77rhDU6ZM0SeffKJHH33UsX369OmqWrWqBg4cWOZ7VKtWTZL022+/OQ1PChSEBS/o37+/li1bVqZrNGrUSAcPuv8CfObMGf3666/69ddfNWXKFD322GOaMGGCx6RqdWmncnTPpxu08Y8Tbvs6NqqqJjX5Zg8wixCbTR0Tqur6y+r6uyneQc8CAl32MWmiOcfMe/T4Himmepkv07FjR11yySVKTEx0hIXDhw9r8eLFuvfeexUZGVnkud99951ycnI87hs8eLCjt+Cmm27Sp59+qjvvvFNr165V37591a5dO0eIMDvCghcYhuF4XbVqVbVv314///yzMjI8zMNdhEOHDkmSmjZtqhtvvFGdO3dW3bp1lZ2dreTkZE2aNEnHjx/Xq6++qtDQUL300kvl/j7MbNMfxzX6kw1KO+0+xm9Mz6Z6+KrmCrXi9IwAzMk1LIRH+6cdAM7bHXfcoUceeURr1qzRFVdcoY8++kh2u73EIUjff/+9vv/+e4/7Lr/8ckdYuP766/Xaa69p/Pjxeu211/Taa69Jkpo0aaJrrrlGDz30kJo1a1a+b6ocUbPgBUOGDNGMGTP0+++/Kz09Xd988815p8eOHTtqyZIl+u233zRhwgT1799f7dq1U5cuXfTPf/5T69atU40aNSRJEydO1N69e73xVkxp9vqDumXKaregUCE8VJNva6vHrr6IoADAt9x6FhiGBASKoUOHKjw8XNOnT5ckJSYmqk2bNiUOGXr55ZdlGIbHP67Dlx555BEdOnRIs2fP1tixY9WlSxf98ccfevfdd3XppZfqq6++8tK7KzvCghfcfffduvXWW9W0adMLvsbPP/+sq6++usjhRU2aNNFzzz0n6WzF/rx58y74XoHiTH6Bnv/qV4374hfl5Rc47atftYLm3tdJ/VrX8VPrAAQ1pk4FAlaNGjXUv39/zZw5U99995127dpVLoXNripVqqSbbrpJkyZN0k8//aQjR47ovvvuU05OjkaNGqW8PHNO0sIwpADWs2dPx2szL+ZRHtIzcnX/jI1avde9+KpTk2p6d0hbVYmJ8EPLAEDULCDwVah6tg4gUFSoWq6XGzVqlObOnasRI0YoKipKt912W7le35O4uDi98847WrRokQ4cOKCtW7eqXbt2Xr/v+SIsBLBz5+QNDbXo3OWSfj10Und/vEF/nsh223dH5wQ93a+FwkLpJAPgR3nOi88xDAkBJySkXAqGA9XVV1+tevXq6c8//9TgwYNVpUoVn9zXZrMpJsbcXy4QFgLYuYuEtGzZ0o8t8Z4FWw7p8S+2KOeM87CjiLAQvTyotW5sF++nlgHAORiGBAS00NBQzZs3TykpKeU+vemUKVPUtm1bdejQwW3fvHnztGPHDlWuXFmXXHJJud63vBAWAlRWVpbeeOMNSVJkZKQGDBhw3tdISUkpdn9qauqFNK1c5BcYmvjNLr2/3L1LtHZslKYMa6fL6lf2fcMAwBOGIQEBr3379mrfvn2pjy9u6tTatWvrnnvukSQtXrxY99xzj5o2beqY3TIzM1ObNm3STz/9pJCQEE2ePLnYaVr9ibAQoJ544gn98ccfks6uCFi37vnPXV6/fv3ybla5OJl9Rg/N3KRlu4647WvfsIomD22rmpWi/NAyACgCYQEIOsVNnXrZZZc5wsIrr7yizp0769tvv9WPP/7o+DK2Xr16uv322/XAAw+YslahkM04d1EAeE2jRo104MCBUq/gXJzPPvtMQ4cOlXR2+NGGDRtUoUKF877O+SzkdvDgQcXHe3nIT+ovOnRglyYv26O0U+5JvWuzGhrcsYHCmRYVgNksHCtlnvMFx62zpIuu8VtzgKL8/vvvstvtCgsLM/Xc/nBXmv92KSkpji+Dy+t3t6DtWSiPFY8TExM1YsSIsjfmPCxbtkyjRo2SdHbBty+//PKCgoIkjytEnys1NVUdO3a8oGtfiIPfvqP6e2fp/yTJ08RGB/7/HwAwO3oWAFhE0IaFQLR+/Xpdf/31ys3NVcWKFfX111+XqbDZ6z0F52Frykn98ttR3cYnEoAVEBYAWETQ/mq2Y8eOMl+jTh3fLQD266+/6pprrtHp06cVGRmpefPm6YorrvDZ/b3tknqxyqpZUXJfRgEAAkt4jFTTmjPUAQg+QRsWWrRo4e8mlNqePXt01VVXKT09XWFhYZo1a5Z69+7t72aVK5vNpnatmip1VV3lnMlXxcgwVasYKcoTAASUSnWk7k9I4Rc2PBQAzCZow0KgSElJUZ8+fZSamqqQkBB99NFHFzRNaiAI6/OcQjuO04rtf2noFQ3Kpa4EAAAAF46wYGJpaWnq06ePY/ak999/X0OGDPFvo7ysZmyUhl3Z0N/NAAAAgKQQfzcAnp04cUJXX321du3aJUmaNGmS7rrrLj+3CgAAAMGEngUv2L17t1asWOG0LSMjw/G/SUlJTvuuueYa1a5d2/Fzbm6urrvuOm3evFmSdNttt6lPnz7atm1bkfeMiYlRQkJC+bwBAAAAQIQFr1ixYoVGjhzpcV96errbvuTkZKewkJqaqp9//tnx82effabPPvus2Ht2795dy5Ytu/BGAwAAywsNDZXdbpfdbld+fr5CQ0P93SSUQn5+vux2uyT5/L8Zw5AAAACCREzM/9YAOXz4sPLz8/3YGpRGfn6+Dh8+7Pj53P+GvkDPgheMGDGiTCs7N2rUSIZhlF+DAAAAJMXGxurYsbOLGp06dUqnTp1SWBi/DppZYY9Cobi4OJ/en08HAABAkKhQoYLq1q2rQ4cOOba5/jIK86pbt66ioqJ8ek/CAgAAQBCJi4tTZGSkTp48qczMTIYimVxoaKhiYmIUFxfn86AgERYAAACCTlRUlF9+8UTgocAZAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARBc4o0rlTqaWmpvqxJQAAACjJub+vldeUuIQFFOnIkSOO1x07dvRjSwAAAHA+jhw5okaNGpX5OgxDAgAAAOCRzTAMw9+NgDnl5ORo69atkqQaNWqwHHwRUlNTHT0va9euVZ06dfzcIgQzPo8wEz6PMJNg+Dza7XbHyJDWrVuXy1oa/PaHIkVFRalDhw7+bkZAqVOnjuLj4/3dDEASn0eYC59HmImVP4/lMfToXAxDAgAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHrEoGwAAAACP6FkAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QF4BxpaWlauHChnnvuOV177bWqXr26bDabbDabRowYcd7XW7x4sQYNGqT4+HhFRkYqPj5egwYN0uLFi8u/8bCc9evX61//+pf69u3r+AxVrFhRzZs318iRI7VixYrzuh6fR1yoU6dOaebMmXr00UfVvXt3NW3aVHFxcYqIiFDNmjXVo0cPvfrqq0pPTy/V9X7++WcNHTpUDRs2VFRUlGrXrq2rr75an3/+uZffCYLBE0884fi322azadmyZSWew/OxGAYAB0lF/rn99ttLfZ38/Hxj1KhRxV7vzjvvNPLz8733ZhDQunbtWuznp/DP8OHDjdzc3GKvxecRZfXtt9+W6vNYvXp1Y8mSJcVea/z48UZISEiR17juuuuM7OxsH70zWM2mTZuMsLAwp89UcnJykcfzfCwZPQtAERo0aKC+ffte0LnPPPOMpk2bJklq06aNPv/8c61du1aff/652rRpI0n68MMP9eyzz5Zbe2Ethw4dkiTVrVtXDz30kL744gutXbtWq1at0uuvv6569epJkj7++OMSe734PKI81K9fX8OHD9ebb76puXPnatWqVVq5cqVmzZqlm266SaGhoTp69Kiuv/56bdmyxeM1pkyZohdeeEEFBQVq0qSJpk2bprVr12revHnq2bOnJGnRokW64447fPnWYBEFBQW6++67ZbfbVbNmzVKdw/OxFPydVgAzee6554wFCxYYhw8fNgzDMPbt23fePQu7du1yfKvRvn17Iysry2l/Zmam0b59e0OSERYWZvz+++/l/TZgAdddd50xa9Ysw263e9x/5MgRo3nz5o7P5/Llyz0ex+cR5aGoz+G5/vvf/zo+j4MGDXLbn56ebsTFxRmSjAYNGhhHjhxxu0f//v1L9W0w4MmkSZMMSUaLFi2Mp556qsTPEs/H0iEsAMW4kLBw7733Os5ZtWqVx2NWrVrlOOa+++4rxxYjmCxYsMDxOXrggQc8HsPnEb500UUXOYYjuXrllVccn7PPP//c4/kHDx40QkNDDUlGv379vN1cWMiBAweMihUrGpKMZcuWGePHjy8xLPB8LB2GIQHlyDAMzZ8/X5LUokULXXnllR6Pu/LKK3XRRRdJkubPny/DMHzWRlhH4bANSdqzZ4/bfj6P8LVKlSpJknJyctz2zZs3T5IUGxurG264weP58fHx6tOnjyTp+++/1+nTp73TUFjO/fffr4yMDN1+++3q3r17icfzfCw9wgJQjvbt2+cYa17Sw6pw/59//qn9+/d7u2mwoNzcXMfr0NBQt/18HuFLu3bt0ubNmyWd/eXrXHl5eVq7dq0k6W9/+5siIiKKvE7hZzE3N1fr16/3TmNhKbNnz9bChQtVtWpV/ec//ynVOTwfS4+wAJSj7du3O167/mPp6tz9O3bs8FqbYF3Lly93vG7ZsqXbfj6P8LasrCz9/vvvev3119W9e3fZ7XZJ0tixY52O++2335Sfny+JzyLK14kTJ/TQQw9Jkl555RVVr169VOfxfCy9MH83ALCSlJQUx+v4+Phij61fv77j9cGDB73WJlhTQUGBJkyY4Pj55ptvdjuGzyO8ISkpSSNHjixy/5NPPqkhQ4Y4beOzCG8ZN26cDh8+rM6dO2vUqFGlPo/PZOkRFoBydO742ooVKxZ7bExMjON1RkaG19oEa5o0aZJjWMcNN9ygdu3auR3D5xG+dPnll+uDDz5Qhw4d3PbxWYQ3/PTTT/rwww8VFham999/XzabrdTn8pksPYYhAeXo3KK+4sbkSlJkZKTjdXZ2ttfaBOtZvny5nnzySUlSzZo19d5773k8js8jvGHgwIHaunWrtm7d6piPftCgQdq8ebNuvfVWLVy40O0cPosob3l5ebr77rtlGIYefvhhXXLJJed1Pp/J0iMsAOUoKirK8TovL6/YY88tTq1QoYLX2gRr+fXXXzVo0CDZ7XZFRUVpzpw5RS4+xOcR3lC5cmVdcskluuSSS9ShQwcNHjxYc+fO1ccff6y9e/dqwIABSkpKcjqHzyLK20svvaSdO3eqQYMGGj9+/Hmfz2ey9AgLQDkqnDZQKrmrMjMz0/G6pC5QQDo7e0ffvn11/PhxhYaGaubMmerWrVuRx/N5hC8NGzZMN910kwoKCjRmzBgdO3bMsY/PIsrTzp079fLLL0uS3n77badhQqXFZ7L0qFkAytG5RVLnFk95cm6R1LnFU4Anhw4dUp8+fXTo0CHZbDZNnz5dAwYMKPYcPo/wtQEDBmj27NnKzMzUkiVLHIXOfBZRniZNmqS8vDw1btxYWVlZmjlzptsx27Ztc7z+4YcfdPjwYUlS//79FRMTw2fyPBAWgHLUqlUrx+udO3cWe+y5+z1NewkUOnr0qK666irt3btX0tlv0oYPH17ieXwe4Ws1atRwvD5w4IDjdfPmzRUaGqr8/Hw+iyizwmFBe/fu1a233lri8f/+978dr/ft26eYmBiej+eBYUhAOUpISFDdunUlOc+B78mPP/4oSapXr54aNWrk7aYhQJ08eVJXX321Y07wCRMm6P777y/VuXwe4Wt//vmn4/W5wzUiIiLUsWNHSdKqVauKHSNe+FmNjIxU+/btvdRSBDuej6VHWADKkc1mcwwN2blzp1avXu3xuNWrVzu+qRgwYMB5TfeG4JGVlaXrrrtOGzdulCQ988wzeuKJJ0p9Pp9H+NqcOXMcr1u3bu20b+DAgZKkU6dOae7cuR7PT0lJ0XfffSdJ6t27t9O4cqBQUlKSDMMo9s+5Rc/JycmO7YW/7PN8PA8GgCLt27fPkGRIMm6//fZSnbNr1y4jNDTUkGS0b9/eyMrKctqflZVltG/f3pBkhIWFGb/99psXWo5Al5uba/Tt29fx+XvooYcu6Dp8HlEeEhMTjezs7GKPef311x2f14SEBMNutzvtT09PN+Li4gxJRsOGDY2jR4867bfb7Ub//v0d10hOTi7vt4EgMn78+BI/SzwfS4eaBeAcK1as0O7dux0/Hz161PF69+7dbtMBjhgxwu0azZs31+OPP64JEyZo/fr16ty5s5544gk1adJEe/bs0SuvvKJNmzZJkh5//HE1a9bMK+8Fge3WW2/V0qVLJUm9evXSqFGjnAr2XEVERKh58+Zu2/k8ojw8//zzevTRR3XjjTeqS5cuatKkiSpWrKjTp09r69at+uyzz7Ry5UpJZz+LH3zwgUJDQ52uUbVqVb3yyiu65557dODAAV1xxRV65pln1Lp1ax06dEhvvPGGkpOTJZ39/Pfo0cPXbxNBhudjKfk7rQBmcvvttzu+iSjNn6Lk5+cbd9xxR7Hnjho1ysjPz/fhu0MgOZ/Pof7/N7VF4fOIsmrYsGGpPofx8fHG0qVLi73Wc889Z9hstiKv0a9fvxJ7MYCSlKZnwTB4PpYGNQuAF4SEhGjatGlatGiRBgwYoLp16yoiIkJ169bVgAED9PXXX+vDDz9USAj/F4T38XlEWX3zzTd67bXXdMMNN+jSSy9VrVq1FBYWpkqVKqlJkya68cYblZiYqF27dumqq64q9lovvPCCVqxYoSFDhqh+/fqKiIhQzZo1ddVVV2nGjBlatGiR04JZgDfxfCyZzTAMw9+NAAAAAGA+wRuTAAAAABSLsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAACzt+eefl81mk81m83dTACDgEBYAIAjt37/f8Qt0Wf4AAKyNsAAA8KmkpCRH2Ni/f7+/m2Npy5Ytc/xdL1u2zN/NARCAwvzdAACA79WrV09bt24tcn/r1q0lSe3bt1diYqKvmgUAMBnCAgAEofDwcF1yySUlHhcTE1Oq4wAA1sQwJAAAAAAeERYAAOetoKBAn376qfr166fatWsrIiJCNWrUUM+ePTV58mTl5eW5nVM4fn7kyJGObQkJCW5F065j61evXq1nn31WPXr0cNwrNjZWrVq10r333qvt27d7++06nD59Wq+99pp69erl1JY2bdrogQce0MqVK4s898iRI3r22WfVpk0bVa5cWVFRUWrUqJGGDRumFStWlHjvH374QbfeeqsSEhJUoUIFRUdHq2HDhrryyiv12GOP6YcffnAcW1jA3rNnT8e2nj17uv1dJyUllenvA0AQMAAAcCHJkGR0797dbV96errRuXNnxzGe/rRs2dLYv3+/03nJycnFnlP4Jzk52XFOYmJiiceHhoYa7777bpHvZfz48Y5jy+Lbb781qlevXmJ7PPnmm2+M2NjYYs+7//77jfz8fI/njx07tsT7VqtWzXH8vn37SvV3nZiYWKa/EwDWR80CAKDU8vPz9fe//12rVq2SJHXv3l1jxoxRQkKCDh06pOnTp2vevHnasWOHevfurc2bN6tixYqSpA4dOmjr1q2aP3++nn32WUnSN998o7p16zrdIyEhwfHabrerSpUqGjBggLp166ZmzZopJiZGhw4d0saNG/XWW2/p6NGjGjNmjFq0aKFevXp55X0nJyfr2muvld1uV2hoqIYNG6YBAwaoQYMGysnJ0fbt27V48WItWLDA7dzNmzerf//+ysvLU3h4uMaMGaPrr79eMTEx2rRpkyZMmKB9+/bp3XffVUxMjF555RWn8xcuXKg33nhDknTppZfq3nvvVcuWLRUXF6cTJ07o119/1Xfffae1a9c6ziksYF+3bp3uuOMOSdL06dPVoUMHp2vHx8eX898UAMvxd1oBAJiPiuhZeOeddxz7hg8fbhQUFLid+/TTTzuOGTdunNv+c3sL9u3bV2w7UlJSjMzMzCL3nzhxwrj00ksNSUaXLl08HlPWnoXs7Gyjbt26hiQjOjraqefD1R9//OG2rUOHDo4ekG+++cZt/7Fjx4xWrVoZkoyQkBBj27ZtTvuHDRtmSDIaNmxonD59ush7p6enu207tzenuHYDQFGoWQAAlNq7774rSapRo4beeecdjwuzvfDCC2rRooUkaerUqcrNzb3g+9WrV0/R0dFF7o+Li9O//vUvSdKKFSuUnp5+wfcqyscff6xDhw5Jkl566SX16NGjyGPr16/v9PPatWu1bt06SdJdd92lvn37up1TpUoVffDBB5LO1oJMnjzZaf/hw4clSW3btnX00nhStWrVkt8MAJwnwgIAoFQOHTqkHTt2SJJuvvlmVapUyeNxYWFhjiLm48ePa+PGjeXWhszMTO3fv1+//vqrtm3bpm3btik8PNyxf8uWLeV2r0ILFy6UdHYa2bvuuuu8zv3uu+8cr0eNGlXkcZ07d1bLli3dzpGkOnXqSJJ+/PFH7dmz57zuDwBlRVgAAJTKtm3bHK+vuOKKYo89d/+5512Io0eP6umnn9ZFF12kSpUqKSEhQZdccolat26t1q1b67rrrnM6trxt2rRJktSuXbtiezk8KXzvERERuvzyy4s9tvDv7Pfff3eaTWr48OGSpPT0dF1yySUaPHiwEhMTtXv37vNqCwBcCMICAKBUjh075nhds2bNYo+tXbu2x/PO14YNG9SiRQu9/PLL+u2332QYRrHHZ2dnX/C9ilIYQAq/4T8fhe+9atWqCgsrfk6Rwr8zwzB0/Phxx/bevXvrnXfeUYUKFZSTk6NZs2bpjjvuULNmzRQfH6977rnHKz0qACARFgAAF8BTrUJ5y8vL080336z09HSFh4frkUce0fLly5WamqqcnBwZhiHDMJyG5pQUJvylrH9f999/v/bv369JkyapX79+iouLkyT9+eefmjJlitq0aeOYYQoAyhNhAQBQKucW0P7111/FHltYlOt63vn44YcftHfvXknS5MmT9dprr6lbt26qXbu2IiMjHceVpeeiNKpXry5JSk1NPe9zC997enq67HZ7sccW/p3ZbDZVqVLFbX/NmjU1duxYLVq0SMeOHdOGDRv07LPPqnLlyjIMQy+++KLmz59/3m0EgOIQFgAApXLJJZc4Xq9Zs6bYY8+d8//c86TSf8v+66+/Ol7fcsstRR63fv36Ul3vQrVt29Zxn6ysrPM6t/C95+XlafPmzcUeW/h31qxZM0VERBR7bEhIiNq2bat///vf+v777x3bZ8+e7XScL3qAAFgbYQEAUCp169Z1zNgze/ZsZWRkeDwuPz9fSUlJks5OC1r4y3ahqKgox+viplU995v4zMxMj8cUFBRo6tSppWr/herfv78kKSsryzHFaWn16dPH8Xr69OlFHrdq1Spt377d7ZzSaNu2raMnwrXAu7R/1wBQFMICAKDU7r//fknSkSNH9OCDD3o85oUXXnD84nvXXXc5DRmSnAuFi5sKtFmzZo7XheHD1VNPPVWuU7N6MnToUNWrV0+S9Mwzz2j58uVFHpuSkuL0c8eOHdW+fXtJZ9ecOLcXoNDJkyc1evRoSWd7DO69916n/bNmzSq2cHv9+vWOguhzV7+WSv93DQBFsRlmrQYDAPhN4fCV7t27a9myZY7t+fn56tq1q1atWiVJ6tWrl+677z4lJCQoNTVV06dP19y5cyVJTZo00ebNm90WEjt9+rRq1qypnJwctW3bVhMmTFDDhg0VEnL2+6t69eqpQoUKyszMVOPGjZWWlqbQ0FDdeeedGjRokKpXr67du3c7fvnu3LmzVq5cKUlKTEzUiBEjnO73/PPP64UXXpB04QXQycnJ6tu3r+x2u8LCwjRs2DANHDhQ8fHxys3N1c6dO/X111/rq6++cvsGf/PmzbriiiuUl5eniIgIPfDAA+rfv79iYmK0adMmTZgwwVGbMW7cOL3yyitO5zdq1EgnT57UgAED1K1bNzVv3lwxMTFKT0/XihUr9Pbbb+vYsWMKDQ3V6tWrHeGkUP369ZWSkqKEhAS98cYbuuiiixQaGipJqlWrVpHrZQCAJMlva0cDAExLkiHJ6N69u9u+9PR0o3Pnzo5jPP1p2bKlsX///iKvP27cuCLPTU5Odhy3ZMkSIyoqqshje/ToYWzbts3xc2Jiotu9xo8f79hfFkuWLDGqVKlS7Psu6h7ffPONERsbW+x5999/v5Gfn+92bsOGDUu8Z2RkpMf3bhiGMXny5CLPK+ocACjEMCQAwHmpWrWqfvzxR3388ce65pprVKtWLYWHh6tatWrq0aOH3nnnHW3evFkNGzYs8hoTJkzQ1KlT1bVrV1WtWtXxTberq6++WuvXr9fQoUNVt25dhYeHq0aNGurevbs++OADff/994qJifHWW3Vry969e/XSSy+pU6dOqlatmkJDQxUbG6u2bdtq7NixToXd5+rbt692796tp59+WpdffrliY2MVGRmpBg0a6LbbbtNPP/2kd955x9G7cq7k5GS9+eabuvHGG9W6dWvVqFFDYWFhio2NVZs2bfTYY49p+/btbj0qhe699159+eWX6tu3r2rWrFnieg8AcC6GIQEAAADwiJ4FAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB79PyHokMOGCn9oAAAAAElFTkSuQmCC" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(1, 1, figsize=(4, 3), dpi=200)\n", + "ax.plot(\n", + " cost_AGP.cpu()[9:],\n", + " best_seen_AGP.cpu()[9:],\n", + " label=\"AGP\"\n", + ")\n", + "ax.plot(\n", + " cost_MES.cpu()[9:],\n", + " best_seen_MES.cpu()[9:],\n", + " label=\"MES\"\n", + ")\n", + "\n", + "ax.set_title(\"Branin\", fontsize=\"12\")\n", + "ax.set_xlabel(\"Total cost\", fontsize=\"10\")\n", + "ax.set_ylabel(\"Best seen\", fontsize=\"10\")\n", + "ax.tick_params(labelsize=10)\n", + "ax.legend(loc=\"lower right\", fontsize=\"7\", frameon=True, ncol=1)\n", + "plt.tight_layout()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-01-29T08:23:44.635831300Z", + "start_time": "2024-01-29T08:23:44.193377300Z" + } + }, + "id": "e3604083ed32e0eb" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/sphinx/source/acquisition.rst b/sphinx/source/acquisition.rst index 77aafef6b9..733f7eb60a 100644 --- a/sphinx/source/acquisition.rst +++ b/sphinx/source/acquisition.rst @@ -143,11 +143,6 @@ Preference Acquisition Functions .. automodule:: botorch.acquisition.preference :members: -Multi Information Source Acquisition Functions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: botorch.acquisition.augmented_multisource - :members: - Objectives and Cost-Aware Utilities ------------------------------------------- diff --git a/sphinx/source/models.rst b/sphinx/source/models.rst index 38854678da..5356c3641b 100644 --- a/sphinx/source/models.rst +++ b/sphinx/source/models.rst @@ -44,11 +44,6 @@ GP Regression Models .. automodule:: botorch.models.gp_regression :members: -Multi Information Source GP Regression Models -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: botorch.models.gp_regression_multisource - :members: - Multi-Fidelity GP Regression Models ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: botorch.models.gp_regression_fidelity @@ -187,4 +182,4 @@ Inducing Point Allocators Other Utilties ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: botorch.models.utils.assorted - :members: + :members: \ No newline at end of file diff --git a/test/acquisition/test_multi_source.py b/test_community/acquisition/test_multi_source.py similarity index 97% rename from test/acquisition/test_multi_source.py rename to test_community/acquisition/test_multi_source.py index 00636de4a8..85a912b276 100644 --- a/test/acquisition/test_multi_source.py +++ b/test_community/acquisition/test_multi_source.py @@ -6,9 +6,9 @@ import torch -from botorch.acquisition.augmented_multisource import AugmentedUpperConfidenceBound +from botorch_community.acquisition.augmented_multisource import AugmentedUpperConfidenceBound from botorch.exceptions import UnsupportedError -from botorch.models.gp_regression_multisource import SingleTaskAugmentedGP +from botorch_community.models.gp_regression_multisource import SingleTaskAugmentedGP from botorch.utils.testing import BotorchTestCase, MockModel, MockPosterior diff --git a/test/models/test_gp_regression_multisource.py b/test_community/models/test_gp_regression_multisource.py similarity index 99% rename from test/models/test_gp_regression_multisource.py rename to test_community/models/test_gp_regression_multisource.py index 285598c0ec..c232bbc817 100644 --- a/test/models/test_gp_regression_multisource.py +++ b/test_community/models/test_gp_regression_multisource.py @@ -13,7 +13,7 @@ from botorch import fit_gpytorch_mll from botorch.exceptions import InputDataError, OptimizationWarning from botorch.models import FixedNoiseGP, SingleTaskGP -from botorch.models.gp_regression_multisource import ( +from botorch_community.models.gp_regression_multisource import ( _get_reliable_observations, FixedNoiseAugmentedGP, get_random_x_for_agp, diff --git a/tutorials/multi_source_bo.ipynb b/tutorials/multi_source_bo.ipynb deleted file mode 100644 index d7afb38dc4..0000000000 --- a/tutorials/multi_source_bo.ipynb +++ /dev/null @@ -1,708 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "source": [ - "## Multi-Information Source BO with Augmented Gaussian Processes\n", - "\n", - "In this tutorial, we show how to perform Multiple Information Source Bayesian Optimization in BoTorch based on the Augmented Gaussian Process (AGP) and the Augmented UCB (AUCB) acquisition function proposed in [1].\n", - "The key idea of the AGP is to fit a GP model for each information source and *augment* the observations on the high fidelity source with those from *cheaper* sources which can be considered as *reliable*. The GP model fitted on this *augmented* set of observations is the AGP.\n", - "The AUCB is a modification of the standard UCB -- computed on the AGP -- suitably proposed to also deal with the source-specific query cost.\n", - "\n", - "We emprically show that the *AGP-based* Multiple Information Source Basyesian Optimization usually performs better than other multi-fidelity approaches [2].\n", - "\n", - "[1] [Candelieri, A., & Archetti, F. (2021). Sparsifying to optimize over multiple information sources: an augmented Gaussian process based algorithm. Structural and Multidisciplinary Optimization, 64, 239-255.](https://link.springer.com/article/10.1007/s00158-021-02882-7)\n", - "[2] [The arxiv will be available soon.](https://arxiv.org/)\n" - ], - "metadata": { - "collapsed": false - }, - "id": "826ca0dcff42a3ba" - }, - { - "cell_type": "code", - "execution_count": 1, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: matplotlib in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (3.8.2)\n", - "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (1.2.0)\n", - "Requirement already satisfied: cycler>=0.10 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (4.46.0)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (1.4.5)\n", - "Requirement already satisfied: numpy<2,>=1.21 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (1.26.0)\n", - "Requirement already satisfied: packaging>=20.0 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (23.2)\n", - "Requirement already satisfied: pillow>=8 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (10.1.0)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (3.1.1)\n", - "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib) (2.8.2)\n", - "Requirement already satisfied: six>=1.5 in c:\\users\\ponti\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n", - "[notice] A new release of pip is available: 23.2.1 -> 23.3.2\n", - "[notice] To update, run: python.exe -m pip install --upgrade pip\n" - ] - } - ], - "source": [ - "!pip install matplotlib" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:37:32.213316900Z", - "start_time": "2023-12-18T11:37:29.151143400Z" - } - }, - "id": "8aa9032dbb2c2b04" - }, - { - "cell_type": "code", - "execution_count": 2, - "outputs": [], - "source": [ - "import os\n", - "import matplotlib.pyplot as plt\n", - "\n", - "import torch\n", - "from gpytorch import ExactMarginalLogLikelihood\n", - "\n", - "import botorch\n", - "from botorch import fit_gpytorch_mll\n", - "from botorch.acquisition import InverseCostWeightedUtility, qMultiFidelityMaxValueEntropy\n", - "from botorch.acquisition.augmented_multisource import AugmentedUpperConfidenceBound\n", - "from botorch.models import AffineFidelityCostModel, SingleTaskMultiFidelityGP\n", - "from botorch.models.gp_regression_multisource import SingleTaskAugmentedGP, get_random_x_for_agp\n", - "from botorch.models.transforms import Standardize\n", - "from botorch.optim import optimize_acqf, optimize_acqf_mixed\n", - "from botorch.test_functions.multi_fidelity import AugmentedBranin" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:37:36.050798200Z", - "start_time": "2023-12-18T11:37:32.214312700Z" - } - }, - "id": "e55defd1ee4a5b0f" - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "initial_id", - "metadata": { - "collapsed": true, - "ExecuteTime": { - "end_time": "2023-12-18T11:37:36.056871Z", - "start_time": "2023-12-18T11:37:36.054311400Z" - } - }, - "outputs": [], - "source": [ - "tkwargs = {\n", - " \"dtype\": torch.double,\n", - " \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n", - "}\n", - "SMOKE_TEST = os.environ.get(\"SMOKE_TEST\", False)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "outputs": [], - "source": [ - "N_ITER = 10 if SMOKE_TEST else 50\n", - "SEED = 3" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:37:36.071646400Z", - "start_time": "2023-12-18T11:37:36.057904700Z" - } - }, - "id": "e316bd291459a135" - }, - { - "cell_type": "markdown", - "source": [ - "### Problem setup\n", - "We consider the augmented Branin multi-fidelity synthetic test problem. It is important to clarify that *augmented* is not about the AGP: here, it has a different meaning. It means that the Branin test function has been modified by introducing an additional dimension representing the fidelity parameter.\n", - "\n", - "The test function takes the form $f(x,s)$ where $x \\in [-5, 10] \\times [0, 15]$ and $s \\in [0,1]$. The target fidelity is 1.0, which means that our goal is to solve $\\max_x f(x,1.0)$ by making use of cheaper evaluations $f(x,s)$ for $s < 1.0$. In this example, we'll assume that the cost function takes the form $5.0 + s$, illustrating a situation where the fixed cost is $5.0$.\n", - "\n", - "Since a multiple information source context is considered, three different sources are considered, with $s = 0.5, 0.75, 1.00$, respectively." - ], - "metadata": { - "collapsed": false - }, - "id": "6b58c67f5dbf329c" - }, - { - "cell_type": "code", - "execution_count": 5, - "outputs": [], - "source": [ - "problem = AugmentedBranin(negate=True).to(**tkwargs)\n", - "fidelities = torch.tensor([0.5, 0.75, 1.0], **tkwargs)\n", - "n_sources = fidelities.shape[0]\n", - "\n", - "bounds = torch.tensor([[-5, 0, 0], [10, 15, n_sources - 1]], **tkwargs)\n", - "target_fidelities = {n_sources - 1: 1.0}\n", - "\n", - "cost_model = AffineFidelityCostModel(fidelity_weights=target_fidelities, fixed_cost=5.0)\n", - "cost_aware_utility = InverseCostWeightedUtility(cost_model=cost_model)" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:37:36.085138100Z", - "start_time": "2023-12-18T11:37:36.071646400Z" - } - }, - "id": "5f13380e681011ea" - }, - { - "cell_type": "markdown", - "source": [ - "### Model initialization\n", - "\n", - "We use a `SingleTaskAugmentedGP` to implement our AGP.\n", - "\n", - "At each Bayesian Optimization iteration, the set of observations from the *ground-truth* (i.e., the highest fidelity and more expensive source) is temporarily *augmented* by including observations from the other cheap sources, only if they can be considered *reliable*. Specifically, an observation $(x,y)$ from a cheap source is considered reliable if it satisfies the following inequality:\n", - "\n", - "$$\\vert\\mu(x)-y\\vert \\leq m \\sigma(x)$$\n", - "\n", - "where $\\mu(x)$ and $\\sigma(x)$ are, respectively, the posterior mean and standard deviation of the GP model fitted on the high fidelity observations only, and $m$ is a technical parameter making more *conservative* ($m→0$) or *inclusive* ($m→∞)$ the augmentation process. As reported in [1], a suitable value for this parameter is $m=1$.\n", - "\n", - "After the set of observations is augmented, the AGP is fitted through `SingleTaskAugmentedGP`.\n" - ], - "metadata": { - "collapsed": false - }, - "id": "81e30344694a9583" - }, - { - "cell_type": "code", - "execution_count": 6, - "outputs": [], - "source": [ - "def generate_initial_data(n):\n", - " train_x = get_random_x_for_agp(n, bounds, 1)\n", - " xs = train_x[..., :-1]\n", - " fids = fidelities[train_x[..., -1].int()].reshape(-1, 1)\n", - " train_obj = problem(torch.cat((xs, fids), dim=1)).unsqueeze(-1)\n", - " return train_x, train_obj\n", - "\n", - "\n", - "def initialize_model(train_x, train_obj, m):\n", - " model = SingleTaskAugmentedGP(\n", - " train_x, train_obj, m=m, outcome_transform=Standardize(m=1),\n", - " )\n", - " mll = ExactMarginalLogLikelihood(model.likelihood, model)\n", - " return mll, model" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:37:36.086286100Z", - "start_time": "2023-12-18T11:37:36.077862900Z" - } - }, - "id": "f8272160f69227ef" - }, - { - "cell_type": "markdown", - "source": [ - "#### Define a helper function that performs the essential BO step\n", - "This helper function optimizes the acquisition function and returns the candidate point along with the observed function values.\n", - "\n", - "The UCB acquisition function has been modified to deal with both the *discrepancy* between information sources and the *source-specific query cost*.\n", - "\n", - "Formally, the AUCB acquisition function, at a generic iteration $t$, is defined as:\n", - "\n", - "$$\\alpha_s(x,\\hat y^+) = \\frac{\\left[\\hat{\\mu}(x) + \\sqrt{\\beta^{(t)}} \\hat{\\sigma}(x)\\right] - \\hat{y}^+}{c_s \\cdot (1+\\vert \\hat{\\mu}(x) - \\mu_s(x) \\vert)} $$\n", - "\n", - "where $\\hat{y}^+$ is the best (i.e., highest) value in the *augmented* set of observations, the numerator is -- therefore -- the optimistic improvement with respect to $\\hat{y}^+$, $c_s$ is the query cost for the source $s$, and $\\vert \\hat{\\mu}(x) - \\mu_s(x) \\vert$ is a discrepancy measure between the predictions provided by the AGP and the GP on the source $s$, respectively, given the input $x$ (i.e., 1 is added just to avoid division by zero).\n", - "\n", - "For more information, please refer to [1]," - ], - "metadata": { - "collapsed": false - }, - "id": "ad21cd7999805d78" - }, - { - "cell_type": "code", - "execution_count": 7, - "outputs": [], - "source": [ - "def optimize_aucb(acqf):\n", - " candidate, value = optimize_acqf(\n", - " acq_function=acqf,\n", - " bounds=bounds,\n", - " q=1,\n", - " num_restarts=5,\n", - " raw_samples=128,\n", - " )\n", - " # observe new values\n", - " new_x = candidate.detach()\n", - " new_x[:, -1] = torch.round(new_x[:, -1], decimals=0)\n", - " return new_x" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:37:36.088896700Z", - "start_time": "2023-12-18T11:37:36.085138100Z" - } - }, - "id": "311309bd6a4d3a92" - }, - { - "cell_type": "markdown", - "source": [ - "### Perform a few steps of multi-fidelity BO\n", - "First, let's generate some initial random data and fit a surrogate model." - ], - "metadata": { - "collapsed": false - }, - "id": "667b55ca7ae58af3" - }, - { - "cell_type": "code", - "execution_count": 8, - "outputs": [], - "source": [ - "torch.manual_seed(SEED)\n", - "train_x, train_obj = generate_initial_data(n=5)" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:37:36.137343Z", - "start_time": "2023-12-18T11:37:36.089845700Z" - } - }, - "id": "a2b9dbfbae2f7d5" - }, - { - "cell_type": "markdown", - "source": [ - "We can now use the helper functions above to run a few iterations of BO." - ], - "metadata": { - "collapsed": false - }, - "id": "ca54230c1481ba60" - }, - { - "cell_type": "code", - "execution_count": 9, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iter 0;\t Fid = 1.00;\t Obj = -12.0999;\n", - "Iter 1;\t Fid = 1.00;\t Obj = -50.0743;\n", - "Iter 2;\t Fid = 0.50;\t Obj = -11.9672;\n", - "Iter 3;\t Fid = 0.50;\t Obj = -8.7046;\n", - "Iter 4;\t Fid = 0.50;\t Obj = -13.3425;\n", - "Iter 5;\t Fid = 1.00;\t Obj = -11.6850;\n", - "Iter 6;\t Fid = 0.50;\t Obj = -8.2926;\n", - "Iter 7;\t Fid = 1.00;\t Obj = -14.4455;\n", - "Iter 8;\t Fid = 0.50;\t Obj = -16.5712;\n", - "Iter 9;\t Fid = 1.00;\t Obj = -32.4978;\n", - "Iter 10;\t Fid = 0.50;\t Obj = -17.8776;\n", - "Iter 11;\t Fid = 1.00;\t Obj = -17.4798;\n", - "Iter 12;\t Fid = 0.50;\t Obj = -14.4403;\n", - "Iter 13;\t Fid = 1.00;\t Obj = -16.5080;\n", - "Iter 14;\t Fid = 0.50;\t Obj = -48.2065;\n", - "Iter 15;\t Fid = 1.00;\t Obj = -67.0580;\n", - "Iter 16;\t Fid = 0.50;\t Obj = -113.7052;\n", - "Iter 17;\t Fid = 1.00;\t Obj = -5.5352;\n", - "Iter 18;\t Fid = 1.00;\t Obj = -14.0937;\n", - "Iter 19;\t Fid = 0.50;\t Obj = -116.5878;\n", - "Iter 20;\t Fid = 1.00;\t Obj = -5.2398;\n", - "Iter 21;\t Fid = 1.00;\t Obj = -2.9797;\n", - "Iter 22;\t Fid = 1.00;\t Obj = -0.9191;\n", - "Iter 23;\t Fid = 1.00;\t Obj = -0.4690;\n", - "Iter 24;\t Fid = 1.00;\t Obj = -2.7735;\n", - "Iter 25;\t Fid = 1.00;\t Obj = -5.6576;\n", - "Iter 26;\t Fid = 1.00;\t Obj = -0.4424;\n", - "Iter 27;\t Fid = 0.50;\t Obj = -144.6051;\n", - "Iter 28;\t Fid = 1.00;\t Obj = -2.1584;\n", - "Iter 29;\t Fid = 1.00;\t Obj = -37.6281;\n", - "Iter 30;\t Fid = 1.00;\t Obj = -2.3379;\n", - "Iter 31;\t Fid = 0.50;\t Obj = -230.5202;\n", - "Iter 32;\t Fid = 0.75;\t Obj = -77.1763;\n", - "Iter 33;\t Fid = 1.00;\t Obj = -3.1912;\n", - "Iter 34;\t Fid = 1.00;\t Obj = -1.5353;\n", - "Iter 35;\t Fid = 1.00;\t Obj = -1.2452;\n", - "Iter 36;\t Fid = 1.00;\t Obj = -1.4735;\n", - "Iter 37;\t Fid = 1.00;\t Obj = -3.5458;\n", - "Iter 38;\t Fid = 0.75;\t Obj = -58.2390;\n", - "Iter 39;\t Fid = 1.00;\t Obj = -4.2493;\n", - "Iter 40;\t Fid = 0.50;\t Obj = -53.5646;\n", - "Iter 41;\t Fid = 0.50;\t Obj = -17.8283;\n", - "Iter 42;\t Fid = 1.00;\t Obj = -21.7001;\n", - "Iter 43;\t Fid = 1.00;\t Obj = -3.9768;\n", - "Iter 44;\t Fid = 1.00;\t Obj = -10.2322;\n", - "Iter 45;\t Fid = 1.00;\t Obj = -1.1175;\n", - "Iter 46;\t Fid = 1.00;\t Obj = -3.9205;\n", - "Iter 47;\t Fid = 0.75;\t Obj = -68.0340;\n", - "Iter 48;\t Fid = 0.50;\t Obj = -38.8294;\n", - "Iter 49;\t Fid = 0.50;\t Obj = -80.6787;\n" - ] - } - ], - "source": [ - "cumulative_cost = 0.0\n", - "\n", - "with botorch.settings.validate_input_scaling(False):\n", - " for it in range(N_ITER):\n", - " mll, model = initialize_model(train_x, train_obj, m=1)\n", - " fit_gpytorch_mll(mll)\n", - " acqf = AugmentedUpperConfidenceBound(\n", - " model,\n", - " beta=3,\n", - " maximize=True,\n", - " best_f=train_obj[torch.where(train_x[:, -1] == 0)].min(),\n", - " cost={i: fid + 5.0 for i, fid in enumerate(fidelities)},\n", - " )\n", - " new_x = optimize_aucb(acqf)\n", - " if model.n_true_points < model.max_n_cheap_points:\n", - " new_x[:, -1] = fidelities.shape[0] - 1\n", - " train_x = torch.cat([train_x, new_x])\n", - "\n", - " new_x[:, -1] = fidelities[new_x[:, -1].int()]\n", - " new_obj = problem(new_x).unsqueeze(-1)\n", - " train_obj = torch.cat([train_obj, new_obj])\n", - "\n", - " print(\n", - " f\"Iter {it};\"\n", - " f\"\\t Fid = {new_x[0].tolist()[-1]:.2f};\"\n", - " f\"\\t Obj = {new_obj[0][0].tolist():.4f};\"\n", - " )" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:39:03.821762700Z", - "start_time": "2023-12-18T11:37:36.108340600Z" - } - }, - "id": "8d02e319798a28d7" - }, - { - "cell_type": "markdown", - "source": [ - "## Comparison to MES" - ], - "metadata": { - "collapsed": false - }, - "id": "ab5e775efb0ccfe9" - }, - { - "cell_type": "code", - "execution_count": 10, - "outputs": [], - "source": [ - "def initialize_mes_model(train_x, train_obj, data_fidelity):\n", - " model = SingleTaskMultiFidelityGP(\n", - " train_x,\n", - " train_obj,\n", - " outcome_transform=Standardize(m=1),\n", - " data_fidelity=data_fidelity,\n", - " )\n", - " mll = ExactMarginalLogLikelihood(model.likelihood, model)\n", - " return mll, model" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:39:03.827771200Z", - "start_time": "2023-12-18T11:39:03.821256200Z" - } - }, - "id": "1c93f20bdaffccac" - }, - { - "cell_type": "code", - "execution_count": 11, - "outputs": [], - "source": [ - "def optimize_mes_and_get_observation(mes_acq, fixed_features_list):\n", - " candidates, acq_value = optimize_acqf_mixed(\n", - " acq_function=mes_acq,\n", - " bounds=problem.bounds,\n", - " q=1,\n", - " num_restarts=5,\n", - " raw_samples=128,\n", - " fixed_features_list=fixed_features_list,\n", - " )\n", - " # observe new values\n", - " cost = cost_model(candidates).sum()\n", - " new_x = candidates.detach()\n", - " new_obj = problem(new_x).unsqueeze(-1)\n", - " return new_x, new_obj, cost" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:39:03.834820500Z", - "start_time": "2023-12-18T11:39:03.827771200Z" - } - }, - "id": "1a1bba8e3496b54d" - }, - { - "cell_type": "code", - "execution_count": 12, - "outputs": [], - "source": [ - "train_x_mes = torch.clone(train_x[:10])\n", - "train_x_mes[:, -1] = fidelities[train_x_mes[:, -1].int()]\n", - "train_obj_mes = torch.clone(train_obj[:10])" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:39:03.840874700Z", - "start_time": "2023-12-18T11:39:03.834820500Z" - } - }, - "id": "e8414dfbd0643afa" - }, - { - "cell_type": "code", - "execution_count": 13, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iter 0;\t Fid = 0.50;\t Obj = -57.5522;\n", - "Iter 1;\t Fid = 0.50;\t Obj = -45.4163;\n", - "Iter 2;\t Fid = 0.50;\t Obj = -9.0258;\n", - "Iter 3;\t Fid = 1.00;\t Obj = -48.1935;\n", - "Iter 4;\t Fid = 0.50;\t Obj = -0.9326;\n", - "Iter 5;\t Fid = 0.50;\t Obj = -80.6658;\n", - "Iter 6;\t Fid = 0.50;\t Obj = -2.1240;\n", - "Iter 7;\t Fid = 0.50;\t Obj = -10.3696;\n", - "Iter 8;\t Fid = 0.50;\t Obj = -4.4469;\n", - "Iter 9;\t Fid = 0.50;\t Obj = -8.9561;\n", - "Iter 10;\t Fid = 0.50;\t Obj = -22.0102;\n", - "Iter 11;\t Fid = 0.50;\t Obj = -16.9033;\n", - "Iter 12;\t Fid = 0.50;\t Obj = -13.5406;\n", - "Iter 13;\t Fid = 0.75;\t Obj = -0.6079;\n", - "Iter 14;\t Fid = 0.50;\t Obj = -31.6025;\n", - "Iter 15;\t Fid = 0.50;\t Obj = -38.2686;\n", - "Iter 16;\t Fid = 0.50;\t Obj = -17.9994;\n", - "Iter 17;\t Fid = 0.50;\t Obj = -19.7949;\n", - "Iter 18;\t Fid = 0.50;\t Obj = -275.5655;\n", - "Iter 19;\t Fid = 0.50;\t Obj = -111.8389;\n", - "Iter 20;\t Fid = 0.50;\t Obj = -91.2188;\n", - "Iter 21;\t Fid = 0.50;\t Obj = -76.4585;\n", - "Iter 22;\t Fid = 0.50;\t Obj = -0.5961;\n", - "Iter 23;\t Fid = 0.50;\t Obj = -14.7103;\n", - "Iter 24;\t Fid = 0.50;\t Obj = -2.7677;\n", - "Iter 25;\t Fid = 0.50;\t Obj = -53.0683;\n", - "Iter 26;\t Fid = 0.50;\t Obj = -1.3054;\n", - "Iter 27;\t Fid = 0.50;\t Obj = -23.7584;\n", - "Iter 28;\t Fid = 0.50;\t Obj = -12.2472;\n", - "Iter 29;\t Fid = 0.50;\t Obj = -12.6918;\n", - "Iter 30;\t Fid = 0.50;\t Obj = -4.2754;\n", - "Iter 31;\t Fid = 1.00;\t Obj = -1.7593;\n", - "Iter 32;\t Fid = 1.00;\t Obj = -44.1689;\n", - "Iter 33;\t Fid = 0.50;\t Obj = -108.9165;\n", - "Iter 34;\t Fid = 0.50;\t Obj = -143.6092;\n", - "Iter 35;\t Fid = 1.00;\t Obj = -3.5225;\n", - "Iter 36;\t Fid = 0.50;\t Obj = -4.6244;\n", - "Iter 37;\t Fid = 0.50;\t Obj = -2.1491;\n", - "Iter 38;\t Fid = 1.00;\t Obj = -189.5238;\n", - "Iter 39;\t Fid = 1.00;\t Obj = -0.9491;\n", - "Iter 40;\t Fid = 1.00;\t Obj = -33.1628;\n", - "Iter 41;\t Fid = 0.50;\t Obj = -86.3697;\n", - "Iter 42;\t Fid = 1.00;\t Obj = -5.4117;\n", - "Iter 43;\t Fid = 1.00;\t Obj = -3.7590;\n", - "Iter 44;\t Fid = 0.50;\t Obj = -12.7589;\n", - "Iter 45;\t Fid = 0.50;\t Obj = -4.9297;\n", - "Iter 46;\t Fid = 1.00;\t Obj = -17.4007;\n", - "Iter 47;\t Fid = 0.50;\t Obj = -46.4645;\n", - "Iter 48;\t Fid = 1.00;\t Obj = -16.0177;\n", - "Iter 49;\t Fid = 1.00;\t Obj = -2.7148;\n" - ] - } - ], - "source": [ - "candidate_set = torch.rand(\n", - " 1000, problem.bounds.size(1), device=problem.bounds.device, dtype=problem.bounds.dtype\n", - ")\n", - "candidate_set = problem.bounds[0] + (problem.bounds[1] - problem.bounds[0]) * candidate_set\n", - "\n", - "cumulative_cost = 0.0\n", - "\n", - "with botorch.settings.validate_input_scaling(False):\n", - " for it in range(N_ITER):\n", - " mll, model = initialize_mes_model(train_x_mes, train_obj_mes, data_fidelity=2)\n", - " fit_gpytorch_mll(mll)\n", - " acqf = qMultiFidelityMaxValueEntropy(\n", - " model, candidate_set, cost_aware_utility=cost_aware_utility\n", - " )\n", - " new_x, new_obj, cost = optimize_mes_and_get_observation(acqf,\n", - " fixed_features_list=[{2: fid} for fid in fidelities])\n", - " train_x_mes = torch.cat([train_x_mes, new_x])\n", - " train_obj_mes = torch.cat([train_obj_mes, new_obj])\n", - " cumulative_cost += cost\n", - " print(\n", - " f\"Iter {it};\"\n", - " f\"\\t Fid = {new_x[0].tolist()[-1]:.2f};\"\n", - " f\"\\t Obj = {new_obj[0][0].tolist():.4f};\"\n", - " )" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:41:01.885178100Z", - "start_time": "2023-12-18T11:39:03.842873900Z" - } - }, - "id": "fcaf20bf41f1b680" - }, - { - "cell_type": "markdown", - "source": [ - "## Plot results" - ], - "metadata": { - "collapsed": false - }, - "id": "dd44be2238fc0110" - }, - { - "cell_type": "code", - "execution_count": 14, - "outputs": [], - "source": [ - "mapping_fid = dict(zip(range(fidelities.shape[0]), fidelities.tolist()))\n", - "cost_AGP = torch.cumsum(torch.tensor([mapping_fid[int(source)] for source in train_x[:, -1].tolist()]), dim=0)\n", - "cost_MES = torch.cumsum(train_x_mes[:, -1], dim=0)" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:41:01.892898500Z", - "start_time": "2023-12-18T11:41:01.887864Z" - } - }, - "id": "5c5cc1ef1808ad1f" - }, - { - "cell_type": "code", - "execution_count": 15, - "outputs": [], - "source": [ - "train_obj[torch.where(train_x[:, -1] != fidelities.shape[0] - 1)] = train_obj.min()\n", - "best_seen_AGP = torch.cummax(train_obj, dim=0)[0]" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:41:01.920142Z", - "start_time": "2023-12-18T11:41:01.893908500Z" - } - }, - "id": "4a7f7020f195a31f" - }, - { - "cell_type": "code", - "execution_count": 16, - "outputs": [], - "source": [ - "train_obj_mes[torch.where(train_x_mes[:, -1] != 1)[0]] = train_obj_mes.min()\n", - "best_seen_MES = torch.cummax(train_obj_mes, dim=0)[0]" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:41:01.921142200Z", - "start_time": "2023-12-18T11:41:01.907620600Z" - } - }, - "id": "f696548b9b3f50a5" - }, - { - "cell_type": "code", - "execution_count": 17, - "outputs": [ - { - "data": { - "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwsAAAJECAYAAABZ37i3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAB7CAAAewgFu0HU+AAB0K0lEQVR4nO3dd3hUddrG8XvSQyqhQ0KXJr2p9CYqLoK6KioiiIquqCgudtHdVVFWUVdRQAyuiiDqgoAgloCCKL1JUUqQQDCQUNKTSc77B29GpiQEksyck3w/15VrZ059BmeTuefXbIZhGAIAAAAAF36+LgAAAACAOREWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAgArWr18/2Ww22Ww2rVy50tflAECpERYAAOfl7A++nn78/PwUERGhJk2aaPjw4Zo1a5bS09N9XTYA4AIQFgAA5cowDGVkZCgxMVGLFi3S3XffrYsuukhffPGFr0sDAJynAF8XAACwrm7duql79+5O2woLC3Xy5Elt3bpVO3fulCT98ccfuu6667Rw4UL95S9/8UWpAIALQFgAAFywIUOG6Nlnny12/5o1azRixAglJSWpoKBA99xzjw4cOKDAwEDvFWkCjFMAYFV0QwIAVJiePXtqwYIFjueHDx/mgzMAWAhhAQBQoS699FI1adLE8byoaxIAwPwICwCAClevXj3H48zMTLf9iYmJjtmUGjdu7Ni+evVq3XnnnWrVqpWioqJks9k0YcIEp3MLCwv1ww8/6JlnntHgwYPVsGFDVatWTcHBwapXr54GDBig559/XsePHy9VrWfP7FRkz549mjBhglq3bq3w8HBFRkaqQ4cOevzxx0t13dJMnTp69GjHMXPmzJEkZWVlafr06erVq5fq1Kmj4OBgxcXF6eabb9aaNWtK9XoAoCwYswAAqHBHjx51PK5bt+45j8/Ly9MDDzygGTNmlHhcfn6+mjRposOHDxd736NHjyohIUEvvvii3nnnHY0cOfK8an/nnXc0YcIE5ebmOm3ftm2btm3bplmzZmn58uXq2rXreV33XHbu3Km//vWv2rVrl9P2pKQkzZs3T/PmzdMzzzyj5557rlzvCwBnIywAACrUhg0btH//fsfz3r17n/Ochx56yBEU2rVrpw4dOigwMFC//vqr/Pz+bBQvKChwBIXw8HBdfPHFatq0qSIjI5Wfn6+kpCT99NNPOn36tDIzM3XbbbcpMDBQN910U6lqnzNnju69915JUsuWLdW1a1eFhoZq9+7dWrNmjQzDUGpqqq655hrt2rVLUVFRpf53KcmRI0c0aNAgJScnKzo6Wr1791bdunV1/Phxfffddzp16pQk6R//+IfatGlT6tcDAOfNAADgPPTt29eQZEgyJk+eXOKx69atMxo3buw4/tprr/V43IEDBxzH+Pv7G5KMuLg44/vvv3c7Nicnx/E4NzfXGDNmjJGQkGDk5eV5vHZOTo7x8ssvGwEBAYYkIzo62khPTy+25qI6JBnBwcFGrVq1jGXLlrkdt2rVKiMyMtJx7HPPPVfsNc/+N0tISPB4zO233+50X0nGo48+amRmZjodl5qaagwYMMBxbNOmTY3CwsJi7w0AZUHLAgDggn355ZduffYLCwt16tQpbdu2TTt27HBsv/baa/Xhhx+e85oFBQWqVq2avvnmG7Vo0cJtf3BwsONxUFCQ3nvvvRKvFxwcrL///e8qLCzUY489ppMnT+qDDz5wtBicyzfffKP27du7be/Tp49eeOEFjR8/XpL08ccf65lnninVNc8lNzdXjz/+uF544QW3fTExMZo7d66aNWumzMxM7d+/X+vWrdMll1xSLvcGgLMRFgAAF2z9+vVav359icfUq1dP06dP1/Dhw0t93fHjx3sMCmUxZswYPfbYY5LOBIDShIW7777bY1AoMmrUKE2YMEF2u1179uzR6dOnFRkZWeZaa9WqVWLwqFOnjq6++mp98sknkkRYAFBhCAsAgAqVnJys66+/XrfccoveeOMNVa9e/ZznjBgx4rzvU1hYqI0bN2rLli1KSkrS6dOnlZ+f7/HYLVu2lOqaN9xwQ4n7IyIi1KxZM+3Zs0eGYejgwYNq167d+ZbuZujQoQoJCSnxmE6dOjnCQmJiYpnvCQCeEBYAABds8uTJHldwzszMVGJiopYtW6aXX35Zx44d04cffqjNmzfrhx9+KDEwBAYGntcHbrvdrjfeeEPTpk1TUlJSqc4p7TSqpamjRo0ajsenT58u1XXNel8AcMU6CwCAchcWFqaLL75YjzzyiDZv3qwGDRpIkn755Rc9/PDDJZ5bvXp1BQSU7rus3NxcXX311Zo4cWKpg4Ikpaenl+q40sxuFBgY6HhcXEvG+fLVfQHAFWEBAFChGjRooMmTJzuef/jhh07rLrgKDQ0t9bWfe+45rVixQtKZxdRuuukmffLJJ9q1a5dOnTqlvLw8GYbh+Cly9uOSnL0wmzf56r4A4IpuSACACnfFFVc4Htvtdq1atarMawPk5ubqP//5j+P5nDlzNGrUqGKPL21rAgDgT7QsAAAqXL169ZyeHzx4sMzXXLdunTIyMiRJF198cYlBobzuCQBVDWEBAFDhsrKynJ6fvQrzhTpy5IjjcWkGBH///fdlvicAVDWEBQBAhdu0aZPT86IBz2VxduBwDSOuCgsLNXPmzDLfEwCqGsICAKDCTZs2zfHYZrNpwIABZb5m06ZNHY9XrVqlU6dOFXvs1KlTtXXr1jLfEwCqGsICAKDCnDx5UuPGjdPixYsd22655RbVqVOnzNfu1KmTo4Xi1KlTuuGGG5y6JklnBkE/88wzeuyxxxQWFlbmewJAVcNsSACAC/bll196XOAsKytLiYmJ+umnn5Sdne3Y3qJFC7366qvlcm8/Pz/985//1B133CFJ+vrrr9WiRQv16NFDjRo1UmpqqlauXKkTJ05IkmbOnKlbb721XO4NAFUFYQEAcMHWr1+v9evXl+rYa665RjNmzFDt2rXL7f5jxozR3r179cILL0g6s3L0119/7XRMSEiIXnvtNd1yyy2EBQA4T4QFAEC5Cw4OVlRUlJo3b65LL71Ut9xyi7p06VIh93r++ed11VVX6c0339Tq1at17NgxRUREKDY2VldeeaXGjh2riy66qELuDQCVnc0o7TKWAAAAAKoUBjgDAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8KCRRw8eFATJ05Uq1atFBYWppiYGHXr1k1Tp05VVlaWr8sDAABAJWQzDMPwdREo2eLFizVy5EidPn3a4/4WLVpo6dKlat68uZcrAwAAQGVGWDC5zZs3q2fPnsrOzlZ4eLgef/xx9e/fX9nZ2Zo3b55mzZol6Uxg2LBhgyIiInxcMQAAACoLwoLJ9enTRz/88IMCAgL0/fff67LLLnPaP3XqVE2aNEmSNHnyZD377LM+qBIAAACVEWHBxNatW6dLLrlEkjRu3Di98847bscUFhaqbdu22rVrl6Kjo5WSkqLAwMByuX9OTo62b98uSapVq5YCAgLK5boAAAAof3a7XceOHZMktWvXTiEhIWW+Jp/+TGzhwoWOx2PGjPF4jJ+fn0aNGqXHH39cJ0+eVEJCggYPHlwu99++fbu6d+9eLtcCAACA96xbt07dunUr83WYDcnEVq9eLUkKCwtTly5dij2ub9++jsdr1qyp8LoAAABQNdCyYGK7du2SJDVv3rzELkCtWrVyO6c81KpVy/F43bp1qlevXrldGwAAAOUrOTnZ0Svk7M9xZUFYMKmcnBwdP35ckhQbG1visdWrV1dYWJgyMzN16NChUt8jKSmpxP1Ffd4kqV69euesAwAAAOZQXmNNCQsmlZ6e7ngcHh5+zuOLwkJGRkap7xEXF3dBtQEAAKBqYMyCSeXk5DgeBwUFnfP44OBgSVJ2dnaF1QQAAICqhZYFkzp7qqu8vLxzHp+bmytJCg0NLfU9ztVl6ex+bwAAAKh6CAsmdfZKzKXpWpSZmSmpdF2WijAGAQAAACWhG5JJhYSEqEaNGpLOPRD5xIkTjrDAOAQAAACUF8KCibVp00aStHfvXtnt9mKP2717t+Nx69atK7wuAAAAVA2EBRPr1auXpDNdjDZu3FjscatWrXI87tmzZ4XXBQAAgKqBsGBiw4cPdzyOj4/3eExhYaH++9//SpKio6PVv39/b5QGAACAKoCwYGLdu3dX7969JUmzZ8/W2rVr3Y555ZVXHKs2P/jggwoMDPRqjQAAAKi8mA3J5F5//XX17NlT2dnZGjx4sJ544gn1799f2dnZmjdvnmbOnClJatGihSZOnOjjagEAAFCZEBZMrlOnTpo/f75Gjhyp06dP64knnnA7pkWLFlq6dKnTdKsAAABAWdENyQKGDh2qbdu26aGHHlKLFi1UrVo1RUdHq2vXrnrppZe0efNmNW/e3NdlAgAAoJKxGYZh+LoImFNSUpJj3YZDhw6xiBsAAICJVcRnN7ohAQAswzAMZecXKD3HrvSc/P//X7sycj0/P/O/duUXFPq6dABVXIs6EfrHsLa+LuO8ERYAAKXy/a/HNPWrPUpJz/H6vQ1DyrUXKiPXroJCGsQBWE+e3ZpfWhAWAADntHjrEU2Yv4UP6gBQxRAWAAAlWrDhkB79bJvICQBQ9RAWAADF+uCng3p64Q5fl1GskEA/hQcHKjIkQOEhAYoICVB4cIAiQgIVERKgiP9/HBTA5H8AfKtWRLCvS7gghAUAgEfv/rBf/1q6y237yEsbanCbul6vJyjA7/8DwJkgEBYcQAgAgApGWAAAuPnPt7/pla9/ddt+T99mevTKlrLZbD6oCgDgbYQFAICDYRia+tUeTV+5z23fw5e30P0DmhMUAKAKISwAACSdCQr/WLJT8WsS3fY9MaSV7u7TzPtFAQB8irAAAFBhoaEnF+7Qx+t+d9v3j2EXa9Rljb1fFADA5wgLAFDF2QsKNenTbfp882Gn7Tab9NJ17XVjtzgfVQYA8DXCAgBUYXn2Qk2Yv1lfbj/qtN3fz6ZXb+ygYR0b+KgyAIAZEBYAoIrKyS/Q+Lmb9M2uFKftgf42/efmzrqyrfenRwUAmAthAQCqoOy8At39wQb98Ntxp+3BAX5657Yu6t+yto8qAwCYCWEBAKqYjFy77ohfr3WJaU7bqwX5691RXdWjeU0fVQYAMBvCAgBUIaey8nV7/DptOXTSaXtEcIDix3RT18YxvikMAGBKhAUAqEImLtjiFhSiQgP1wdjuah8b7ZOa4COGIe1eIiVvPfMYlVuHm6WazX1dBSyIsAAAVcTB1Ey3wcw1w4P0wdhL1LpepI+qgs9890/ph1d8XQW8peFlhAVcEMICAFQRGxJPOD2PCAnQvLsvU/Pa4T6qCD7zxy/S6mm+rgKABfj5ugAAgHds/N05LPRsVpOgUBUZhrT8Mcko9HUlACyAlgUAqCI2urQsdGlU3UeVwKd2L5UOfO+8reFlUmR939QD74io4+sKYFGEBQCoAk5l5+vXlHSnbZ0JC1VPfo604knnbZGx0sjPpaBqvqkJgKnRDQkAqoAth046TXgTFOCntg0Y1Fzl/DRdOpHovG3wPwgKAIpFWACAKmDjQecuSO0bRCk4wN9H1cAnTidL3//beVvDy6SLr/NNPQAsgbAAAFXApoOMV6jyvv2HlJ951gabdOUUyWbzWUkAzI+wAACVXEGhoc0uMyExXqGKSdoobZ3rvK3TSKl+R5+UA8A6CAsAUMntPnpamXkFTts6NyQsVBmFhdKySc7bgiKkgc/4ph4AlkJYAIBKzrULUqMa1VQrIthH1cDrti+QDm9w3tZ3khRe2zf1ALAUwgIAVHKug5sZr1CF5GZI30x23hbTTLrkHt/UA8ByCAsAUMm5rtxMWKhCVk+T0pOdt13xghQQ5Jt6AFgOYQEAKrGU0zk6lJbttI2wUEWcSJR+/I/ztmYDpRZX+KQcANZEWACASmyTS6tCRHCALqod4aNq4FUrnpYKcv98bvOXrnyRqVIBnBfCAgBUYhsSncNCx4bR8vfjw2Kld+B7adcXztu63y3VaumbegBYFmEBACoxxitUQQV2afnjzttCY6R+j/qmHgCWRlgAgEoqJ79AOw6fctrWtVGMj6qB12x6X/pjh/O2AU9JoQRFAOePsAAAldSOw6eUX2A4nvvZpA5xUT6sCBUu+4T03b+ct9VpK3UZ7ZNyAFgfYQEAKinX9RVa1o1UREigj6qBV6x8ScpOc9525YuSn79v6gFgeYQFAKik3Bdji/ZNIfCOY3uk9bOct7UeKjXp45t6AFQKhAUAqIQMw2Dl5qrEMM4Mai60/7nNP1i6/J++qwlApUBYAIBK6GBqllIz85y2Mbi5EvtthbTvW+dtPcZLMU18Uw+ASoOwAACVkGurQq2IYMVWD/VRNahQ9jz3qVLD60q9HvZNPQAqFcICAFRCbusrNKwuGyv3Vk7rZkhp+5y3Xf6cFBzum3oAVCoBvi4AxUtMTNTixYu1cuVKbdu2TYcPH1ZhYaFq1qyprl27asSIEfrrX/+qgAD+MwJwtsnK4xUKC6Xtn0jJ2yQZ5zy8ytv8ofPzBl2ldjf6phYAlQ6fMk3q6aef1vPPPy/DcP9DefjwYR0+fFiLFi3Sq6++qk8//VQNGzb0QZUAzOhUdr72/JHutK2zlcLCyhek76f6ugrruuolyY+OAwDKB79NTCo5OVmGYSgsLEwjR45UfHy8Vq9erQ0bNuiDDz5Qt27dJEnr16/XoEGDlJGR4eOKAZjFlkMndfb3DEEBfmrbINJ3BZ2vXxb6ugLr6nCzFNvV11UAqEQICyZVo0YNvfTSS0pOTtYHH3yg0aNHq2fPnurSpYtGjhyptWvX6sYbzzQz//bbb3r11Vd9XDEAs3Ad3Ny+QZSCAyy0KFf2iXMfA3fVakoDJ/u6CgCVDN2QTOqll14qcb+/v7+mT5+uhQsXKi8vT59++qmeeeYZL1UHwMwsPV5BkvIynZ+3+otUjWlfS1StptTxVimynq8rAVDJEBYsrEaNGmrfvr02bNigffv2nfsEAJVeQaGhzS4zIVlqvEJhgWTPdt424Gmpdivf1AMAVRzdkCwuNzdX0pmWBgDYczRdmXkFTts6N7RQWMjPct8WVM37dQAAJBEWLC0lJUW7du2SJLVu3drH1QAwg40H05yeN6pRTbUign1UzQXI8xQWWC8AAHyFbkgWNnXqVNntdklyDHY+H0lJSSXuT05OvqC6APiO6+DmLlZqVZCkPA8zuwXSsgAAvkJYsKiff/5Zr732miQpNjZW995773lfIy4urpyrAuBrbis3N7ZaWHAZ3GzzlwIs1DICAJUM3ZAs6I8//tBf//pX2e122Ww2vf/++6pWjW/egKou5XSODqU5Dw623ExIrmMWgsIlm803tQAAaFkoK1s5/BGLj4/X6NGjS3Vsenq6rr76akcXoilTpmjAgAEXdN9Dhw6VuD85OVndu3e/oGsD8L5NLq0KEcEBuqh2hI+quUCu3ZAY3AwAPkVYsJCcnBwNGzZMGzdulCQ98sgjmjRp0gVfLzY2trxKA2ACruMVOjaMlr+fxb6Vdx3gHBTmmzoAAJIIC2VWNBtRWdSrd+5FdOx2u2688UYlJCRIku68805NnTq1zPcGUHlssPpibJL7mAUGNwOATxEWyqhVq4pfKKiwsFC33XabFi9eLEm66aabNGPGjAq/LwDryMkv0I7Dp5y2WTMsuHZDYtpUAPAlBjhbwLhx4zRv3jxJ0tChQ/Xhhx/Kz4//dAD+tOPwKeUXGI7nfjapY1y07wq6UG4DnOmGBAC+xCdOk3v44Yf17rvvSpIGDhyoBQsWKCCABiEAzlzHK7SsG6mIkEAfVVMGrt2QGOAMAD5FWDCxZ599VtOmTZMk9ejRQ4sWLVJwMPONA3Dnthhbo2jfFFJWbmGBbkgA4Et8RW1S//nPf/Tcc89Jkho0aKCXX35ZBw4cKPGcli1bKjDQgt8kAigTwzDcpk215HgFiQHOAGAyhAWT+uyzzxyPDx8+rF69ep3znAMHDqhx48YVWBUAMzqYmqXjGXlO27o0jPFRNWXk1rLAmAUA8CW6IQGAxbl2QaoZHqy4mFAfVVNGnlZwBgD4DC0LJrVy5UpflwDAIja6dEHq2qh6uawu7xOs4AwApkLLAgBY3KbKsBhbEVZwBgBTISwAgIWdzsnXnj/SnbZ1tnRYcB3gTFgAAF8iLACAhW3+/aSMP9diU5C/n9o2iPRdQWXl1g2JsAAAvkRYAAALcx3c3C42SsEB/j6qphy4DXBmzAIA+BJhAQAsrFKNV5BYlA0ATIawAAAWVVBoaHNlWYxNkgoLPbQs0A0JAHyJsAAAFrXnaLoy8wqctnVuaOGw4BoUJFZwBgAfIywAgEW5rq/QqEY11YoI9lE15cC1C5JENyQA8DHCAgBY1MbENKfnXazcqiBJ+Z7CAi0LAOBLhAUAsCjXlgVLr68gubcs2PykgBDf1AIAkERYAABLSjmdo0Np2U7bLD24WfKwenO4ZLP5phYAgCTCAgBY0iaXVoWI4AC1qBPho2rKieuCbAxuBgCfIywAgAW5LsbWsWG0/P0s/i0806YCgOkQFgDAglzDguW7IEkeFmSjZQEAfI2wAAAWk5NfoB2HTzttqxxhwaUbEtOmAoDPERYAwGJ2HD6lvIJCx3ObTeoYF+27gsqL2wBnuiEBgK8RFgDAYly7ILWsE6GIkEAfVVOOXLshMcAZAHyOsAAAFuMaFro2rgRdkCT3RdnohgQAPkdYAAALMQzDbdrUSjFeQWKAMwCYEGEBACzkUFq2jmfkOW3r0jDGR9WUM7ewwJgFAPA1wgIAWMjWpJNOz2PCghQXE+qbYsqb25gFwgIA+BphAQAsZJtLWGgfGyWbzeKLsRWhZQEATIewAAAWsjXplNPz9rHRvimkIrCCMwCYDmEBACyioNDQjsPOYaFDbJSPqqkAtCwAgOkQFgDAIvYdy1BWXoHTtkrVsuC2gjNhAQB8jbAAABax9dBJp+cNokNVKyLYN8VUBNcVnFmUDQB8jrAAABaxzW28QiXqgiR56IbEomwA4GuEBQCwCPeZkKJ9UkeFKCxkgDMAmBBhAQAsINdeoJ3Jp522VarBzfZsSYbzNlZwBgCfIywAgAXsTk5XfoHzh+m2lSksuHZBkuiGBAAmQFgAAAtw7YLUtFaYIkMCfVNMRfAUFhjgDAA+R1gAAAtwXYytQ2UaryB5CAs2KTDUJ6UAAP5EWAAAC3Af3FyJuiBJHgY3h0s2m29qAQA4EBYAwOQyc+3am+K8YFmlmglJ8rAgG12QAMAMCAsAYHI7Dp9S4VljmwP8bLq4fqTvCqoIbmssMG0qAJgBYQEATM51MbYWdSIUEujvo2oqiNvqzYQFADADwgIAmNxWl/EKHeIq2XgFyUM3JMICAJgBYQEATM61ZaHSjVeQPAxwZswCAJgBYQEATOxEZp5+T3P+IF3pZkKSGLMAACZFWAAAE9t22LlVITjATy3qRPiomgrkFhZYvRkAzICwAAAmtu3QSafnF9ePVKB/JfzV7RoWWL0ZAEyhEv7FAYDKw3Xl5ko5XkGiGxIAmBRhwYKWLVsmm83m+Hn22Wd9XRKACuK6cnOlnAlJkvIJCwBgRoQFi8nMzNS9997r6zIAeMHRUzlKSc912kbLAgDAmwgLFvP000/r4MGDql27tq9LAVDBXNdXiAgOUJMalfRDtOuibIQFADAFwoKFbNy4UW+88YaCg4P1/PPP+7ocABXMtQtSu9go+fnZfFNMRXMb4ExYAAAzICxYREFBge666y4VFBToiSeeUPPmzX1dEoAKViUWYyvCCs4AYEqEBYuYNm2aNm/erBYtWujRRx/1dTkAKphhGG5hoUNlXIytCCs4A4ApERYsIDExUZMnT5Ykvf322woODvZxRQAq2sHULJ3Kznfa1j4u2jfFeAOLsgGAKREWLODee+9VVlaWbr31Vg0YMMDX5QDwAtfBzTXDg1Q/KsQ3xVQ0w2A2JAAwqQBfF4CSzZ07V8uXL1d0dLReffXVcr12UlJSifuTk5PL9X4ASs/TeAWbrZIObs7PlmQ4b2MFZwAwBcKCiaWlpemhhx6SJL344ovlPl1qXFxcuV4PQPlxnQmpfWUer+DaqiDRDQkATIJuSCb2yCOPKCUlRZdcconuvvtuX5cDwEvsBYXacfi007YOlXkmJNfVmyUGOAOASdCyUEbl0S0gPj5eo0ePdtq2cuVKxcfHy9/fX++88478/Mo/1x06dKjE/cnJyerevXu53xdAyX5LyVB2foHTtqrVsmCTAkJ9UgoAwBlhwYRyc3M1btw4SdIDDzygjh07Vsh9YmNjK+S6AMrGtQtSg+hQ1QivxLOgeVq9uQK+IAEAnD/CQhnt2rWrzNeoV6+e0/PPP/9cv/76qwIDA9WmTRvNmzfP7ZydO3c6Hu/YscNxzCWXXKImTZqUuSYAvrPVdX2FuErcqiC5L8jG4GYAMA3CQhm1atWq3K+Zm5srScrPz9ddd911zuM/++wzffbZZ5LOdGkiLADW5j64OdondXgN06YCgGnRzgsAJpKTX6DdyelO2yr1eAXJw+rNhAUAMAvCggmNHj1ahmGU+JOQkOA4fvLkyY7trgOlAVjLruTTshf+ueaAzSa1a1DJw4JrNyTCAgCYhs+6IZ0+fVrp6ekqKCg457ENGzb0QkUA4Huui7E1rRmmiJBAH1XjJa4DnBmzAACm4dWw8PXXX2v69OlavXq10tLSSnWOzWaT3W6v4MoAwBy2uoxXqNTrKxRhzAIAmJbXwsIDDzygt956S5JkGMY5jgaAqsm1ZaHSj1eQPHRDYvVmADALr4SFuXPn6s0335QkhYSEaPjw4erSpYtiYmIqZLExALCijFy79h1z/uDcPi7aN8V4k9sAZ7ohAYBZeCUszJgxQ5IUFxen7777Ts2aNfPGbSu1fv360UIDVDLbk07p7P9bB/jZ1KZepO8K8ha6IQGAaXnla/1t27bJZrNp8uTJBAUAKIbr+got60YoJNDfN8V4k2tYCCQsAIBZeCUs5OfnS5I6derkjdsBgCW5j1eI9k0h3kbLAgCYllfCQuPGjSVJGRkZJR8IAFWY60xIHeOqwOBmiUXZAMDEvBIWrrvuOknSt99+643bAYDlpGbkKulEttO2qtOywKJsAGBWXgkLEydOVMOGDfXaa69p9+7d3rglAFjKtsPOXZBCAv10Ue0qMoUo3ZAAwLS8EhaioqL01VdfqU6dOurRo4emT5+uEydOeOPWAGAJ2w45h4W29aMU4F9FppZmBWcAMC2vTJ3atGlTSVJWVpZOnjyp+++/Xw888IBq1qypatVK/qNgs9m0b98+b5QJAD7jOhNSlemCJHloWagiLSoAYAFeCQuJiYlOzw3DkGEYSklJOee5NputgqoCAHMwDENbXWZC6lBVBjcbhpRPNyQAMCuvhIXbb7/dG7cBAEtKPpWj4xm5TtuqTMuCPUcyCp23sYIzAJiGV8JCfHy8N24DAJbk2gUpMiRAjWtUkQ/Mrl2QJLohAYCJVJHRcwBgXq5dkNrHRledLpiewgIDnAHANAgLAOBj7oObq8h4BYmwAAAm55VuSK6ys7O1ceNGHT16VFlZWRo+fLgiIyN9UQoA+FRhoaFtHloWqgzX1ZsDq0l+fI8FAGbh1bBw6NAhPfHEE1qwYIHy8/Md27t27ao2bdo4ns+ePVszZsxQVFSUVqxYUXWa4wFUOYmpmUrPsTttqzIzIUms3gwAJue1r29+/vlnderUSXPnzlVeXp5j+lRPhg4dqm3btum7777TihUrvFUiAHida6tCrYhg1Y0M8VE1PsDqzQBgal4JCydPntSwYcOUlpamunXravr06dq+fXuxx9euXVtXXXWVJGnp0qXeKBEAfGLLoZNOzzvERlWt1lS31ZsJCwBgJl7phvTGG28oJSVFNWvW1Nq1a9WwYcNznjNo0CAtWrRI69at80KFAOAbVXrlZoluSABgcl5pWVi8eLFsNpsefvjhUgUFSbr44oslSfv27avI0gDAZ/ILCvXLkdNO26rUTEiS+wBnFmQDAFPxSljYu3evJKlPnz6lPqd69eqSpNOnT5/jSACwpl//SFeu3Xn14qrXsuA6ZoEF2QDATLwSFnJyciRJgYGBpT4nM/PMH5DQ0NAKqQkAfM11cHNcTKhiwoJ8VI2P0A0JAEzNK2Ghdu3akqQDBw6U+pwtW7ZIkurXr18RJQGAz1X58QqShwHOdEMCADPxSli45JJLJEnLli0r1fGGYWjWrFmy2Wzq3bt3RZYGAD6z9ZBzy0KHqjZeQWLqVAAwOa+EhVtvvVWGYeijjz5ytBiUZOLEidq6dask6fbbb6/g6gDA+3LyC7Tnj3SnbVWyZSGfsAAAZuaVsDBs2DD1799fdrtdAwcO1Ntvv62UlBTHfrvdriNHjmjBggXq3bu3Xn/9ddlsNl133XXq0aOHN0oEAK/65chpFRT+uTClzSa1bUDLAmEBAMzFK+ssSNJnn32mgQMHavPmzRo/frzGjx/vWHioU6dOTscahqFLL71Uc+bM8VZ5AOBVruMVmtcKV3iw134lmwdhAQBMzSstC5IUHR2ttWvX6vHHH1dkZKQMw/D4ExoaqkmTJmnlypUKC+OPBoDKyXUmpCrZBUlyDwus4AwApuLVr7GCgoL0/PPP64knntCqVau0YcMGpaSkqKCgQDVq1FCnTp00aNAgRUVVwaZ4AFVGTn6BNh484bStQ1wV/b1HywIAmJpP2rzDwsI0ZMgQDRkyxBe3BwCfWbP3uJ7833b9nuY8ZWiVbVlgBWcAMLUq2EEWALwvLTNP/1q6U59vOuy2LyI4QK3rRfigKhNgBWcAMDWfhIXs7Gxt3LhRR48eVVZWloYPH67IyEhflAIAFcowDH2+6bD+tXSnTmTlu+0P8LPpmaFtFBzg74PqfMww6IYEACbn1bBw6NAhPfHEE1qwYIHy8//8o9m1a1e1adPG8Xz27NmaMWOGoqKitGLFCsesSQBgJYnHM/Xkwu1aszfV4/7ODaP14nXt1bJuFW1VsOdKRoHzNlZwBgBT8VpY+Pnnn3X11VfrxIkTMoyz5xZ3DwJDhw7Vfffdp/z8fK1YsUJXXHGFt8oEgDLLLyjUzO/3641vf1OuvdBtf0RwgCZd1Uq3dm8oP78q/GWIa6uCRDckADAZr0ydevLkSQ0bNkxpaWmqW7eupk+fru3btxd7fO3atXXVVVdJkpYuXeqNEgGgXGw8eEJ/eWO1pn61x2NQuKptXX0zsa9uu7RR1Q4KkvvqzRIDnAHAZLzSsvDGG28oJSVFNWvW1Nq1a9WwYcNznjNo0CAtWrRI69at80KFAFA2p3PyNXX5Hn3480Gd1XjqUC8qRP8Y1laXt6nj/eLMylPLAt2QAMBUvBIWFi9eLJvNpocffrhUQUGSLr74YknSvn37KrI0ACiz5TuOavIXO/TH6Vy3fTabNLpHY00c3LJqrtBckjyXaVMDQiW/KjjQGwBMzCt/ufbu3StJ6tOnT6nPqV69uiTp9OnTFVITgKojYU+KVu5OUV6Be7egsjqUlq3Ve4973Ne6XqSmXNdOHeKiy/2+lUJehvNzZkICANPxSljIycmRJAUGBpb6nMzMM83ToaGhFVITgKohYXeKxsxZ79V7hgT66aFBLXRHryYK9PfK0DBrYtpUADA9r/wVq127tiTpwIEDpT5ny5YtkqT69etXREkAqoivfjnq1fv1aVFLXz/UV+P6NiMonIvb6s2EBQAwG6/8JbvkkkskScuWLSvV8YZhaNasWbLZbOrdu3dFlgagkjue4T6OoCLUCAvS6yM66v0x3RQXwyDdUqEbEgCYnle6Id1666369NNP9dFHH+nBBx9Ux44dSzx+4sSJ2rp1q2w2m26//XZvlAigknJdNbln8xpqUad8F0FrWitcQ9vXU3S1oHK9bqXnOsCZmZAAwHS8EhaGDRum/v37KyEhQQMHDtS//vUvXX/99Y79drtdR44c0Zo1a/TGG2/oxx9/lM1m03XXXacePXp4o0QAldSJzDyn57d0b6Sr29fzUTVw4jZmgQXZAMBsvDaP32effaaBAwdq8+bNGj9+vMaPH+9YvblTp05OxxqGoUsvvVRz5szxVnkAKqkTWc5hoXq10k+0gApGNyQAMD2vjb6Ljo7W2rVr9fjjjysyMlKGYXj8CQ0N1aRJk7Ry5UqFhfGHo0hmZqbeeustDRw4UA0aNFBwcLDq1Kmjzp076/7779eKFSt8XSJgOgWFhk5mO3dDoquQibgNcKYbEgCYjVdXCAoKCtLzzz+vJ554QqtWrdKGDRuUkpKigoIC1ahRQ506ddKgQYMUFRXlzbJMLyEhQWPGjNHBgwedtqekpCglJUWbN2/WDz/8oMGDB/uoQsCcTmfnu62mHBNGWDANuiEBgOn5ZDnRsLAwDRkyREOGDPHF7S3lm2++0dChQ5WTk6Po6Gjdc8896tevn2rXrq2srCzt2rVLS5Ys0R9//OHrUgHTSXPpgiRJ0XRDMg/XsMAAZwAwHZ+EBZTOsWPHNGLECOXk5Khjx45avny56tSp43RMz549deeddyovz/1DEVDVnXQJC9WC/BUS6O+jauCGRdkAwPRMExb++OMPLVmyRMePH1eTJk30l7/8RdWqVe1vmR5//HGlpqaqWrVqWrhwoVtQOFtQEF0rAFdpmc7jFaozXsFcCAsAYHpeCQu7du3S5MmTZbPZNGPGDEVHRzvt/+KLL3TLLbcoOzvbsS02NlaLFi0655oMldWJEyc0d+5cSdLIkSPVqFEjH1cEWI/bTEhhdEEylXzCAgCYnVdmQ1q4cKE+/fRTHTlyxC0opKSkaOTIkcrKynKaFenQoUMaOnSoMjIyPF+0kluyZIkjPF1zzTWO7VlZWdq7d6+OHj0qw3XkJgAnrmss0LJgMrQsAIDpeSUsfPvtt7LZbPrLX/7itm/69OnKyMhQQECAXn31VW3dulUvv/yy/Pz8dOTIEc2aNcsbJZrOTz/95Hjcrl07rV+/XoMHD1ZERIQuuugi1atXT3Xq1NH48eMZ3AwUw3X1ZsKCybit4ExYAACz8Uo3pN9//12S++Jr0pnF2mw2m0aNGqUJEyZIOvPh+LffftOsWbP0xRdf6KGHHvJGmaayc+dOx+OEhATdeeedstvtTsccO3ZMb731lj777DMtX75cHTp0OK97JCUllbg/OTn5vK4HmI17ywLdkEyFlgUAMD2vtCykpKRIkmrXru20/fjx4/rll18kSbfccovTvqKuN2d/aK5K0tLSHI/vuece2Ww2/etf/9Lvv/+u3Nxc/fLLLxo9erQk6ejRoxo+fLhOnz59XveIi4sr8ad79+7l+ZIAr3Mfs0DLgmkYhocVnKv2pBYAYEZeCQtFfe9zcnKctq9evVrSmZl8evXq5bSvXr16kqSTJ09WfIEmlJn55zduOTk5mj17tp588knFxcUpKChIbdq0UXx8vO6++25JUmJiot5++21flQuYkltYoBuSeRTkSUaB8zYWZQMA0/FKWIiJiZH0Z3ekIt9++60kqWvXrm5TfxZ1uQkPN/cfD5vNVuafOXPmuF03JCTE8bh9+/a67bbbPN7/hRdeUHBwsCRp/vz551X7oUOHSvxZt27deV0PMBu3MQu0LJiHaxckiW5IAGBCXhmz0KFDB3399deaO3eubrzxRklnWhsWLFggm82mAQMGuJ1z8OBBSSpxbYHKLCIiwvF48ODBxR5Xo0YNde3aVWvWrNHWrVuVl5dX6jUXYmNjy1wnYGaMWTAxT2GBFZwBwHS8EhZGjBihFStWaPHixRoxYoR69eql+fPnKyUlRX5+frr55pvdzvn5558lyfTrC+zatavM1yjqcnW2uLg4x4xIcXFxJZ5ftL+wsFBpaWmqW7dumWsCrK6w0NDJbGZDMi1aFgDAErwSFkaNGqX33ntPq1ev1oIFC7RgwQLHvjFjxqhVq1Zu53z++eey2Wzq0aOHN0q8YJ5qLw8XX3yx49+poKCgxGPP3h8QYJpFuQGfSs+xq6DQeS0SuiGZiGtYCAiR/Px9UwsAoFheGbPg5+enZcuW6eGHH1ZsbKwCAgIUFxenp59+2uOg3CVLligxMVGSNGTIEG+UaDp9+vRxPN6/f3+Jx+7bt0/SmXEOReNDgKrOdXCzJMXQsmAerN4MAJbgta+hw8LC9O9//1v//ve/z3lsz549deDAAUnm74ZUUfr06aNatWrp2LFjWrx4sV577TX5+7t/63bgwAFt2bJF0pl/Nz8/r+Q/wPTSXMJCcICfQoP45to0WGMBACzBlJ8sq1evrkaNGlXZoCBJ/v7+euSRRySdGez9z3/+0+0Yu92uv/3tbyosLJR0Zj0GAGecdAkLMXRBMhfXsMDqzQBgSqYMCzjjgQceUOfOnSVJzz33nG6++WYtX75cmzZt0oIFC9SnTx8tX75c0pnuWtdff70vywVMJS3TeXBzNF2QzIWWBQCwBEbDmlhISIiWLFmioUOHauPGjZo3b57mzZvndtyQIUM0b9482Ww2H1QJmJN7ywLTpppKfpbzc1ZvBgBTomXB5OrVq6effvpJ77zzjvr27atatWopMDBQdevW1TXXXKPPP/9cS5cudVqXAYD7AGdaFkwmL8P5Oas3A4Ap0bJgAQEBARo3bpzGjRvn61IAy3DthsRMSCZDNyQAsARaFgBUSq7dkFi92WTyXLohsXozAJgSYQFApZSW6RIWmA3JXNxaFuiGBABmRFgAUCmdzHLuhlSdbkjm4rYoGy0LAGBGhAUAlZLromy0LJgMYxYAwBIICwAqHcMwGLNgdnRDAgBL8EpYaNKkiZo1a6a9e/eW+pzff/9dTZs2VbNmzSqwMgCVUUauXfkFhtM2uiGZjNsKznRDAgAz8srUqQcPHpTNZlNeXt65D/5/+fn5SkxMZKExAOfNdbyCRDck06EbEgBYAt2QAFQ6rjMhBfn7KSzI30fVwCO3FZwJCwBgRqYNC6dOnZIkVatG0zSA8+O+enMgrZRm47aCM2EBAMzItGHhww8/lCQ1atTIx5UAsBrXsBBDFyTzYcwCAFhChYxZGDBggMftY8aMUVhYyd8e5ebmav/+/UpJSZHNZtPgwYMrokQAldiJTOcxC9HMhGQu9jyp0O68jdmQAMCUKiQsrFy5UjabTYbx52wkhmFo/fr153Wdpk2b6vHHHy/v8gBUcrQsmJxrFySJbkgAYFIVEhb69Onj1D941apVstls6tKlS4ktCzabTSEhIapXr5569OihESNGnLMlAgBcuY9ZICyYiuvgZokVnAHApCqsZeFsfn5nhkbMmTNHbdq0qYhbAoCDazekGMKCubiOV5CkQL4YAgAz8so6C6NGjZLNZlP16tW9cTsAVZyn2ZBgIq7dkPyDJX+v/DkCAJwnr/x2njNnjjduAwCS3NdZYMyCyeSxxgIAWIWpvsrZt2+fjh8/rsaNG6tOnTq+LgeARbmu4Fydbkjm4rZ6MzMhAYBZeWWdhZSUFE2fPl3Tp093LLZ2tr1796pLly5q0aKFevTooQYNGuj666/XiRMnvFEegErEMAyluXRDqk7Lgrnku4YFBjcDgFl5JSx8/vnnGj9+vF5//XVFRUU57cvNzdVVV12lLVu2yDAMGYahwsJCLVy4UMOGDfNGeQAqkez8AuXZC522VWfMgrm4tSzQDQkAzMorYWHFihWy2Wy69tpr3fbNmTNH+/btkyRdc801ev311zV06FAZhqE1a9Zo/vz53igRQCXhOl5BomXBdFi9GQAswythYc+ePZKkSy+91G3f3LlzJZ1Z9XnhwoW6//77tWjRIg0aNEiGYWjevHneKBFAJeE6XiHAz6aIYFMNzwJjFgDAMrwSFo4dOyZJio2NddqenZ2tn376STabTXfffbfTvjvuuEOStGnTJm+UCKCScG1ZiK4W5LRIJEyAbkgAYBleCQsnT548czM/59v99NNPys/Pl81m06BBg5z2NWnSRNKZwdEAUFquaywwXsGEXFdwZoAzAJiWV8JCePiZJuajR486bS9a6blNmzZuC7YFBp75Ax8QQPcBAKV3IpOZkEzPdVE2uiEBgGl5JSy0atVKkrR8+XKn7Z999plsNpv69u3rdk5RsGC9BQDn44TbGgu0LJiO66JsDHAGANPyytf2V199tX766SfNnDlTrVu3Vu/evTVnzhzt3LlTNptN1113nds5RWMVGjRo4I0SAVQSrt2QWL3ZhBizAACW4ZWwMH78eE2fPl3JyckaP368077LLrtM/fv3dztn8eLFstls6tatmzdKBFBJuLYsRLN6s/kQFgDAMrzSDSkqKkrffPONOnfu7Fh4zTAM9e7dW5988onb8Vu3btX69eslSZdffrk3SgRQSbiNWaAbkvm4reBMWAAAs/La6OHWrVtrw4YNOnDggI4ePap69eqpcePGxR4fHx8v6cz6CwBQWu6zIdGyYDq0LACAZXh9qqEmTZo4pkUtTocOHdShQwcvVQSgMnFvWSAsmI7bAGfCAgCYlVe6IQGAt7jNhsQAZ/NxmzqVsAAAZuX1loXCwkIlJCRo7dq1Onr0qLKysvT888+rXr16jmPy8vJkt9vl7++v4OBgb5cIwKJy8guUnV/gtI0xCybk1g2JqVMBwKy8GhaWLFmiBx54QAcPHnTa/sgjjziFhXfffVf333+/wsPDdeTIEYWF8a0TgHNzHa8gMXWq6djzpELn1h8WZQMA8/JaN6RZs2Zp2LBhSkxMlGEYqlGjhgzD8HjsnXfeqaioKGVkZOh///uft0oEYHFpLuMV/GxSZAgtC6biOhOSRDckADAxr4SF3377Tffdd5+kM7Mb7dy5UykpKcUeHxQUpOuvv16GYWjFihXeKBFAJXDSwxoLfn42H1UDj1wHN0us4AwAJuaVsDBt2jTZ7XZdfPHF+vLLL9WqVatzntO7d29J0ubNmyu6PACVhGvLQjTjFczHdbyCRMsCAJiYV8LCd999J5vNpgkTJigoqHT9h5s3by5JOnToUEWWBqASOekyZiGGaVPNx3UmJP8gyZ9QBwBm5ZWwkJSUJEnntXZC0aDmrCwPTdYA4EFapns3JJhMvsvvdFoVAMDUvBIWbLYzfYbP54N/amqqJCkqKqpCagJQ+bjOhhQTxjfWpuM2bSozIQGAmXklLDRo0ECStH///lKfs3r1aklS06ZNK6QmAJWPa1hg9WYTcg0LDG4GAFPzSljo16+fDMPQ+++/X6rjT506pXfeeUc2m00DBgyo4OoAVBas3mwBbi0LdEMCADPzSlgYN26cbDabVq1apTlz5pR4bGpqqoYPH66jR48qICBA99xzjzdKBFAJnMh0bVmgG5LpEBYAwFK8EhY6deqkBx98UIZhaOzYsbrpppv0ySefOPb/+OOPmjt3ru677z41b95c33//vWw2m55++mk1atTIGyWa2vLlyzVixAg1bdpU1apVU0hIiOLi4jRs2DDNnz9fhYWFvi4RMAW6IVmA66JshAUAMLUAb93olVdeUW5urt5++219+umn+vTTTx0Dn8eNG+c4rmhV5wkTJuipp57yVnmmlJubq1tvvVWfffaZ276kpCQlJSXpiy++0FtvvaUvvvhC0dHR3i8SMBG3lgW6IZkPLQsAYCleaVmQzsyI9NZbb+mrr75Sv379ZLPZZBiG048kXXbZZVq6dKleffVVb5VmWg888IAjKNSuXVv//ve/9d133+mHH37Q9OnTHa0uP/zwg0aMGOHLUgGfy7MXKjOvwGkbLQsm5LqCMwOcAcDUvNayUOTyyy/X5ZdfrvT0dG3evFkpKSkqKChQjRo11LFjR9WsWdPbJZnSH3/8oXfffVeSVL16dW3cuFGxsbGO/b169dKtt96qDh06KDExUV999ZU2bNigrl27+qpkwKdcF2STGLNgSq6LsjF1KgCYmtfDQpGIiAj16dPHV7c3vZ9//tkxFmHMmDFOQaFIZGSkHnroIT344IOSpLVr1xIWUGWluYQFm02KCiUsmI5bNyRaFgDAzLzWDQnnJy/vzw8+Ja010axZM4/nAFXNCZfVmyNDAhXgz68402EFZwCwFNP8JT1x4oSOHTvmGLtQ1bVs2dLxuKTF7Pbt2+fxHKCqcV+9mfEKpuS2KBthAQDMrELDgt1u144dO7Rx40YdO3bMbX9OTo6eeeYZxcbGqmbNmqpbt64iIiL017/+Vb/88ktFlmZ67dq1U48ePSRJc+bM0ZEjR9yOSU9P12uvvSbpTOvD4MGDvVkiYCquYSGa8QrmxGxIAGApFTJmwTAMTZ48WW+88YbS09Md2y+77DJNmzZN3bp1U15enq644gqtXr3acY4kZWVl6X//+5+WLVumL774QgMHDqyIEi0hPj5eV155pQ4cOKDOnTtr0qRJ6ty5swICArRjxw69/PLLOnDggGrWrKmPPvpIQUHn901qUlJSifuTk5PLUj7gVa7TpsYwE5I5ERYAwFIqJCyMGTNGH3zwgSQ5dSv68ccfdeWVV+rnn3/W9OnT9cMPP0iSYmJidNFFF8lut2vnzp3Kzs5Wdna2br31Vu3Zs0dRUVEVUabptWjRQuvXr9fbb7+tl156SRMnTnTaHxgYqEceeUQPPvigxwHQ5xIXF1depQI+dyLLecxCNGHBnBizAACWUu7dkBISEvTf//5XkhQcHKzrr79ejzzyiG644QaFhobq5MmTmjZtmubMmaPAwEDNnDlTx44d09q1a7V+/XodP35cjzzyiCTp2LFjmjNnTnmXaCmLFy/WRx99pIyMDLd9+fn5+uSTTzR37lzGeqDKc2tZCKMbkim5TZ1KWAAAMyv3sBAfHy/pzCJiGzdu1IIFC/Tyyy9r/vz52rhxo+rUqaOZM2fq1KlTeuihh3TnnXc6VnKWpNDQUL388su64oorZBiGli5dWt4lliubzVbmn+IC0cSJEzVmzBjt3r1bw4cP15o1a5SRkaHs7Gxt2rRJY8aM0e+//65HH31Uf/3rX1VQUODxOsU5dOhQiT/r1q0rh38hwDvcxyzQsmBKbgOcmToVAMys3MPCzz//LJvNpoceekitW7d22teqVSs99NBDjg+1t912W7HXuf322yWpyg50PnsV69GjR+t///ufevToobCwMIWEhKhTp05677339PTTT0uSPv/8c02fPv287hEbG1viT7169cr9dQEVJc2lGxKzIZlQQb5U4DLFM4uyAYCplfuYhaJZey677DKP+8/e3rx582Kvc9FFF0mS0tLSyrG68rdr164yX8PTh/Ki1ZttNpv+9a9/FXvuE088oWnTpikjI0Pvvfee7r///jLXA1iR6wrOrN5sQq6tChLdkADA5Mo9LGRmZspmsykmJsbj/ujoaMfj4ODgYq8TEhIiyfwLjbVq1apCrlsUQmrXrq0GDRoUe1xISIguvvhi/fzzz9q9e3eF1AJYQVqma1igZcF0XAc3S6zgDAAmV2HrLJw9DqE02+EsIOBMjrPb7ec8Nj8/3+kcoKrJLyhUeo7z/1eq0w3JfDy1LLAoGwCYmmlWcIazJk2aSJJSU1NL7OqUlpamHTt2OJ0DVDUnXcYrSLQsmJLrTEh+gVIA/50AwMwICyY1dOhQx+MJEyZ47I5VWFioBx54wLHvL3/5i9fqA8zEdbyCxArOppTHGgsAYDUV1m9l+vTpql27ttv2lJQUx+N//OMfxZ5/9nFV0ejRo/Xaa69p165dWrFihbp27ar7779fHTp0kL+/v3bu3Km3335ba9eulSTVqVNHDz/8sI+rBnzDdbxCREiAAv35LsR03FZvZiYkADC7CgsLb7/9drH7isYtPPfccxV1e8sLCgrSsmXLNGzYMG3dulXbt2/X3Xff7fHYJk2a6PPPP1fNmjW9XCVgDq6rN9MFyaTyXcMCg5sBwOwqJCywmnD5aNSokdavX6958+bp008/1aZNm3Ts2DEZhqGYmBi1b99ew4cP16hRoxQWRnM+qi7XBdkY3GxSbi0L/N4CALMr97CQkJBQ3pes0gIDA3XbbbeVuIAdUNW5hQXGK5iT2+rNhAUAMLtyDwt9+/Yt70sCQIlOuIxZiKEbkjnRsgAAlsMIQACW5zpmIZqwYE5uYYExCwBgdoQFAJbn1rIQRjckU3JdwZmWBQAwPcICAMtzHbNAy4JJuS7KxtSpAGB6hAUAlufaDSmG2ZDMyW2AM92QAMDsCAsALM+9ZYFuSKbECs4AYDmEBQCWVlBo6FQ2LQuW4NYNibAAAGZHWABgaaey8+W6DiQrOJsUA5wBwHIICwAsLc1lJiSJbkimxToLAGA5hAUAlnbSZbxCWJC/ggP8fVQNSsQKzgBgOYQFAJbm2rLAtKkmRssCAFgOYQGApZ1k2lTrYAVnALAcwgIAS0tj2lRrKLBLBbnO21iUDQBMj7AAwNJc11igZcGk8jPdt9ENCQBMj7AAwNJOuIxZYNpUk3JdkE1iBWcAsADCAgBLO+EyZoGwYFKu4xUkuiEBgAUQFgBYmlvLQhhjFkzJdfVmvwApgGAHAGZHWABgaa5jFmhZMClWbwYASyIsALA0uiFZBAuyAYAlERYAWFZhoeG2gjPdkEyKBdkAwJIICwAs63ROvgoN5220LJgUYQEALImwAMCyXLsgSYQF0yIsAIAlERYAWFaay0xIIYF+Cg3y91E1KJHromyEBQCwBMICAMtyHa8QQ6uCebkNcGZBNgCwAsICAMtybVmIJiyYl+sKzizIBgCWQFgAYFknXcYsxIQRFkzLdVE2uiEBgCUQFgBYluuCbNHVmDbVtNwGONMNCQCsgLAAwLJcwwItCybGCs4AYEmEBQCWdSLTuRsSYxZMzLUbEis4A4AlEBYAWFaa22xIdEMyLbcBzoQFALACwgIAy3KdOrU63ZDMi0XZAMCSCAsALCvNpRsSqzebGGEBACyJsADAkgzDcG9ZICyYFys4A4AlERYAWFJ6rl32QsNpW/UwxiyYltsKzoQFALACwgIASzrp0gVJomXBtAoLJHuO8zZaFgDAEggLACzJdSakoAA/VQvy91E1KJFrq4JEWAAAiyAsALAk1wXZqlcLlM1m81E1KBFhAQAsi7AAwJJOZDK42TJcV2+WCAsAYBGEBQCWdCKLaVMtw3X1Zpu/5M9/LwCwAsICAEtybVmIYUE283JbvTlcossYAFgCYQGAJbmOWYiuxrSppuW2IFs139QBADhvhAUAluQaFmhZMDEWZAMAyyIsALCkEy7rLEQzZsG83FoWCAsAYBWEhQqQkZGh77//Xv/+97914403qkmTJrLZbLLZbGrcuPF5X2/Hjh0aN26cmjVrptDQUNWqVUu9e/fWO++8I7vdXv4vALAA95YFuiGZFqs3A4BlBfi6gMpo6NChWrlyZblca9asWRo/frzy8v78YJSTk6PVq1dr9erVio+P19KlS1WzZs1yuR9gFe5jFmhZMC1aFgDAsmhZqACGYTgex8TEaPDgwQoPDz/v63z55Ze65557lJeXpzp16uiNN97Qzz//rGXLlum6666TJK1bt07XXnutCgoKyq1+wOwMw3DrhhRDWDAvBjgDgGXRslABbrnlFo0bN07dunVT8+bNJUmNGzdWRkbGOc78U35+vu6//34VFhYqMjJSa9asUbNmzRz7r7zySt13332aPn26Vq9erQ8++ECjR48u75cCmFJWXoHyCgqdtrHOgom5LsoWdP5fngAAfIOWhQpw99136+abb3YEhQvxv//9T/v375ckPf74405BocjUqVNVvXp1x2OgqkhzWWNBkqozZsG8XBdloxsSAFgGYcGkFi5c6HhcXItBtWrVdOONN0qSdu7cqV9//dULlQG+d9Jl9eYAP5vCg2koNS23Ac50QwIAqyAsmNTq1aslSS1btlTdunWLPa5v376Ox2vWrKnwugAzSHMZ3Fw9LEg2VgQ2L08rOAMALIGwYEIZGRk6dOiQJKlVq1YlHnv2/l27dlVoXYBZnHQNC6zebG5u3ZBoWQAAq6Dd3oSSkpIcj2NjY0s8Ni4uzvG4KGBcyH08SU5OPq/rAd7iOmaBwc0m5zbAmTELAGAVhAUTSk9Pdzw+15SrYWF//tE9n9mWJOegAVjJCZcxC4QFk3ObOpVuSABgFXRDMqGcnBzH46Cgkj8EBQcHOx5nZ2dXWE2AmZxwbVkIIyyYmms3JAY4A4BlVNmWhfIYDBkfH18haxuEhIQ4Hp+9crMnubm5jsehoaHndZ9zdVtKTk5W9+7dz+uagDe4rt7MmAWTcxvgTDckALCKKhsWzCwiIsLx+FxdizIz/2zeP99Vos81HgIwK9ewEEPLgrm5dUMiLACAVVTZsFAeMwfVq1evHCpx16BBA8fjcw1CPrt1gDEIqCpOZDqPWYhmzIJ5FRZIdpcukoQFALCMKhsWzjUlqS9FREQoLi5Ohw4d0u7du0s89uz9rVu3rujSAFNwb1mgG5Jpuc6EJDFmAQAshAHOJtWrVy9J0p49e3T06NFij1u1apXjcc+ePSu8LsAMXMMCLQsm5toFSWI2JACwEMKCSQ0fPtzxeM6cOR6PycrK0ieffCJJatOmjVq0aOGFygDfys4rUE5+odM2pk41MY9hgW5IAGAVhAWTuvbaa9W0aVNJ0osvvqh9+/a5HfP3v/9dJ06ccDwGqgLXVgVJiiEsmJdrWLD5SQHBno8FAJhOlR2zUJH27t2r1atXO20rmtUoIyPDraXgyiuvVN26dZ22BQYG6j//+Y+GDh2q06dPq2fPnnrqqafUvXt3nThxQrNmzdJnn30m6UyXpdtuu63iXhBgIq6rN/vZpIgQfpWZltvqzeFSOUxdDQDwDv7CVoDVq1drzJgxHvelpqa67UtISHALC5I0ZMgQvfPOOxo/frz++OMP3X///W7HdO/eXf/73//k7+9fPsUDJnfSw+rNfn58+DQtFmQDAEujG5LJ3XXXXdq4caPuuusuNW3aVCEhIapRo4Z69eqlt99+W2vWrFHNmjV9XSbgNWlug5uZCcnUWGMBACyNloUKMHr06HJd2blt27aaOXNmuV0PsLKTLMhmLazeDJhSdna2Tp8+rczMTBUUFPi6HJTA399fYWFhioyMVGhoqNfvT1gAYCmuYxaYNtXkXLshERYAnzt16pSOHDni6zJQSna7Xbm5uUpLS1P9+vUVFRXl1fsTFgBYiuuYBWZCMjm3Ac6EBcCXsrOz3YJCQAAfB83Mbrc7Hh85ckTBwcEKCQnx2v15dwCwFLeWBVZvNjfXMQsMcAZ86vTp047HkZGRqlu3LpOkmFxBQYGOHj3q+G936tQpr4YFBjgDsBTXdRZoWTA5twHOrN4M+FJm5p//nyQoWIO/v7/TrJln/zf0BsICAEtxDQus3mxyzIYEmErRYOaAgACCgoX4+/s7uot5e0A6YQGApZzIdFlngdmQzM0tLNANCQCshLAAwFLcWxYYs2BqnlZwBgBYBmEBgGXk2guUlefc/ErLgsmxgjMAWBphAYBluE6bKjFmwfRYlA0ALI2wAMAyXKdNtdmkqFC6IZkasyEBMLE77rhDNptNNWrUUG5ubonHbtmyRffcc4/atGmjyMhIBQUFqW7durr88sv1yiuv6NixY27n2Gw2p5+AgADVq1dPw4cP1/fff19RL6tcsc4CAMtwHa8QFRoofz+bj6pBqbit4Ew3JADmkJ6erk8++UQ2m01paWlauHChbrrpJrfjCgsLNWnSJL3yyivy9/dXnz59NHjwYIWFhSklJUVr167VI488osmTJ2vPnj1q0KCB0/k1atTQ+PHjJUk5OTnasmWLFi1apC+++ELz58/XDTfc4JXXe6EICwAsw3UmJNZYsABWcAZgUvPnz1dmZqYefvhhvfbaa5o9e7bHsPDkk0/qlVdeUefOnTV//nw1b97c7ZhNmzbp0UcfVXZ2ttu+mjVr6tlnn3Xa9u677+quu+7SpEmTTB8W6IYEwDJcWxaimQnJ/NxWcCYsADCH2bNnKyAgQJMmTVL//v317bff6uDBg07H/Prrr5o6dapq1aql5cuXewwKktS5c2d9/fXXaty4canufccddygsLEyJiYkeuy+ZCS0LACzjhMuYhRhmQjK3wkJaFgCLKCw03L6QMbPq1YLkV4ZuqDt37tRPP/2kIUOGqE6dOho1apS+/fZbxcfHO7UCvP/++yooKNC4ceNUq1atc163aOG082Gzmbs7LWEBgGWccJkNKZpuSObmGhQkxiwAJnUiK09d/vWNr8sotY1PDVKN8OALPn/27NmSpNtuu02SdN111+lvf/ub4uPj9cwzz8jP70znm7Vr10qS+vfvX8aKnb3//vvKzMxUkyZNVLNmzXK9dnkjLACwDNdvvWhZMDnXLkgSsyEB8Ln8/Hx98MEHioyM1PDhwyVJ4eHhuvbaa/Xhhx/qm2++0eDBgyVJR48elSTVr1/f7TorV67UypUrnbb169dP/fr1c9p2/PhxR2tFTk6Otm7dquXLl8vPz09Tp04t19dWEQgLACyDMQsWk+8pLNANCYBvLVq0SMeOHdPYsWMVEhLi2D5q1Ch9+OGHmj17tiMslGTlypV67rnn3La7hoXU1FTHcf7+/qpZs6aGDRumiRMnqnfv3mV7MV7AAGcAluE2ZoFuSObm1rJgkwJCPB4KAN5S1AVp1KhRTtsHDhyoBg0aaNGiRUpLS5Mk1alTR5J05MgRt+s8++yzMgxDhmHo448/LvZ+LVu2dBxnt9t19OhRLVy40BJBQaJlAYCFMGbBYtxWbw4/s5IeANOpXi1IG58a5OsySq36Bf7+P3TokFasWCFJ6tu3b7HHffjhh3rggQfUo0cPrVy5UgkJCRowYMAF3dPqCAsALIPZkCyGBdkAy/Dzs5VpwLBVzJkzR4WFherVq5datmzptt9ut+v999/X7Nmz9cADD+j222/XlClTNHPmTD344IOmH4xcEQgLACwhv6BQ6bl2p23VGbNgbq7dkBivAMCHDMNQfHy8bDab3n//fTVt2tTjcb/++qvWrl2rDRs2qGvXrpo0aZKmTJmiq666Sh9//LHHtRZOnjxZwdX7DmEBgCV4mv+7Oi0L5sYaCwBM5LvvvtOBAwfUt2/fYoOCJI0ZM0Zr167V7Nmz1bVrVz3//PPKy8vTq6++qlatWqlPnz7q0KGDqlWrppSUFG3btk3r1q1TeHi4Onbs6L0X5CUMcAZgCSddxitIUnQoLQum5toNidWbAfhQ0cDm0aNHl3jcTTfdpNDQUH388cfKzs6Wn5+fXnnlFW3atEljx45VcnKy3n33XU2dOlWLFy9WeHi4pk6dqn379jmmYq1MaFkAYAlpLuMVIkMCFODP9x2m5jbAmbAAwHfmzp2ruXPnnvO4yMhIZWW5LyrZqVMnzZgx47zuaRjGeR1vRvylBWAJJ126IdEFyQLcxiwwwBkArIawAMAS0jKduyFd6LR58CK32ZBYvRkArIawAMASXAc4MxOSBTDAGQAsj7AAwBJc11igG5IFuHZDCqQbEgBYDWEBgCW4rt5MNyQLcBuzQDckALAawgIAS3DthsTqzRbAAGcAsDzCAgBLcA0L0YxZMD/GLACA5REWAFiC65iFGLohmZ/bmAXCAgBYDWEBgCW4jlmIJiyYn9vUqYQFALAawgIA07MXFOpUtnNYYMyCBbCCMwBYHmEBgOm5BgWJdRYswW2AM2EBAKyGsADA9FwHN0t0QzK9wkIGOANAJUBYAGB6ruMVwoMDFBTAry9Ts2dLMpy3sSgbAFgOf20BmF6a2+rNdEEyPdcuSBKLsgGABREWAJjeSZduSKzebAEewwLdkAD4XmJiomw2m2w2m+rWrSu73e7xuF27djmOa9y4sWP7nDlzHNuL+xk9erTTtTIzM/XCCy+oc+fOCg8PV3BwsGJjY9W7d289/vjj2rdvXwW+4rIJ8HUBQHn5Zucf+v63Y8ovMM59MCxl99HTTs8JCxbgFhZsUmCoT0oBAE8CAgL0xx9/6Msvv9Q111zjtn/27Nny8yv+e/WBAweqV69eHvd17NjR8Tg9PV29evXStm3b1Lx5c40cOVI1atTQ8ePHtW7dOk2ZMkXNmjVTs2bNyvyaKgJhAZZXWGjopeW7NeP7/b4uBV7CTEgW4Glws83mm1oAwIMePXpo69ateu+999zCgt1u14cffqhBgwZp1apVHs8fNGiQHnvssXPe57XXXtO2bdt05513aubMmbK5/C48cOCAcnNzL/yFVDC6IcHScu0FmjB/C0GhiqkZHuzrEnAurguyMbgZgMmEhoZqxIgRWrp0qVJSUpz2LVmyRH/88YfuuOOOMt9n7dq1kqT77rvPLShIUpMmTdSqVasy36ei0LJQATIyMrRp0yatW7dO69at0/r165WYmChJatSokeNxSQoLC7V69WotX75cP/74o3bv3q20tDSFhISoYcOG6tOnj+655x61b9++Yl+MiZ3Kztc9H2zU2v2pvi4FXuTvZ9OVbev6ugycC2ssANZSWChlp/m6itILjZFK6CJUWnfccYdmzJihDz74QBMnTnRsf++99xQTE6Phw4eX+R41atSQJP36669O3ZOsgrBQAYYOHaqVK1eW6RqNGzfWoUOH3Lbn5+frl19+0S+//KIZM2bokUce0ZQpUzwm1cos+VS2Rr+3Xnv+SHfaHuBn0w1dYxUc4O+jylCRQgL9Nah1bXVtHOPrUnAubqs3MxMSYGrZadJUc/aZ9+jv+6SwmmW+TPfu3dW2bVvFx8c7wsLRo0e1bNky3XvvvQoOLr4l+5tvvlFOTo7HfSNGjHC0Ftxwww368MMPdeedd2rdunUaPHiwunTp4ggRZkdYqACG8ecA25iYGHXt2lU//vijMjIySjjL2ZEjRyRJzZs31/XXX6+ePXuqfv36ys7OVkJCgqZNm6YTJ07o5Zdflr+/v1544YVyfx1mtedoukbHr1PyKef/g4YF+Wv6yC7q26KWjyoD4ODaDSmIbkgAzOmOO+7Qww8/rJ9//lmXXHKJ3n//fdnt9nN2Qfr222/17bffetzXsWNHR1i45ppr9Morr2jy5Ml65ZVX9Morr0iSmjVrpiuvvFIPPvigLrroovJ9UeWIMQsV4JZbbtHcuXP122+/KTU1VV999dV5p8fu3btr+fLl+vXXXzVlyhQNHTpUXbp0Ua9evfT0009r/fr1qlXrzIfiqVOnav/+qtFn/8d9x/XXd350Cwq1IoI1f9xlBAXALFi9GYBFjBw5UoGBgXrvvfckSfHx8erUqdM5uwy9+OKLMgzD449r96WHH35YR44c0SeffKIJEyaoV69e+v333/XWW2+pffv2+uKLLyro1ZUdYaEC3H333br55pvVvHnzC77Gjz/+qCuuuKLY7kXNmjXTM888I+nMiP2FCxde8L2s4outRzT6vfVKz3GeD7lprTB9fm8PtW0Q5aPKALhxHbPAAGcAJlWrVi0NHTpU8+bN0zfffKM9e/aUy8BmVxEREbrhhhs0bdo0/fDDDzp27Jj+9re/KScnR2PHjlVeXt65L+IDdEOysP79+zsem3kxj7IyDEOzftivF77c7bavS6PqendUV1UPY959wFTcuiExZgEwtdCYM+MArCK0fMeujR07Vp9//rlGjx6tkJAQ3XrrreV6fU+ioqL05ptvaunSpTp48KC2b9+uLl26VPh9zxdhwcLOnpPX379yDugtKDT0zyU7NefHRLd9V1xcR6+P6KSQwMr52gFLcxvgTDckwNT8/MplwLBVXXHFFWrQoIEOHz6sESNGqHr16l65r81mU1iYuX8/EhYs7OxFQlq3bu3DSipGTn6BHpq/Rct2HHXbd/tljfTM0Ivl71e1ZoECLMNt6lS6IQEwL39/fy1cuFBJSUnlPr3pjBkz1LlzZ3Xr1s1t38KFC7Vr1y5FR0erbdu25Xrf8kJYsKisrCy99tprkqTg4GANGzbsvK+RlJRU4v7k5OQLKa1cnMzK053vb9CGgyfc9j12VSuN69O0yk0XC1hKvmtYoBsSAHPr2rWrunbtWurjS5o6tW7durrnnnskScuWLdM999yj5s2bO2a3zMzM1ObNm/XDDz/Iz89P06dPL3GaVl8iLFjUo48+qt9//13SmRUB69evf97XiIuLK++yysWhtCyNjl+nfcecP2wE+tv07xs6aFjHBj6qDECpMcAZQCVX0tSpHTp0cISFl156ST179tTXX3+t77//3vFlbIMGDXT77bfr/vvvN+VYhSI24+xFAVBhGjdurIMHD5Z6BeeSfPTRRxo5cqSkM92PNm7cqNDQ0PO+zvl8M3/o0CHFxsae9z3OS/I2Hdy/W//57jedzs532hUa6K97+zVTq7qRFVsDgPLx7T+k43v+fH71q1K3sb6rB4Ak6bfffpPdbldAQICp5/aHu9L8t0tKSnJ8GVxen92qbMtCeXRhiY+P1+jRo8tezHlYuXKlxo498wc3JiZGn3322QUFBUkeV4g+W3Jysrp3735B174QR76drkZ7P9a/JcnT5EarPGwDYA0McAYAS6qyYcGKNmzYoGuuuUa5ubkKDw/Xl19+WaaBzRXeUnAetiWd1I49KbqFiY2AyomwAACWVGXDwq5du8p8jXr16pVDJaXzyy+/6Morr1R6erqCg4O1cOFCXXLJJV67f0VrWz9K6TWqSSd9XQmA8meT6nf2dREAgAtQZcNCq1atfF1Cqe3bt0+XX365UlNTFRAQoPnz52vgwIG+Lqtc+fnZdEnbFkpeW185+QUKCw5QzfBgMTMqYHHVakiX3SdFMTEBAFhRlQ0LVpGUlKRBgwYpOTlZfn5+ev/99y9omlQrCBj0jMJ6Pa4l6w5pbK8m8iMpAAAA+BRhwcRSUlI0aNAgx+xJ77zzjm655RbfFlXBIkMCdVefpr4uAwAAAJL8fF0APDt58qSuuOIK7dlzZurBadOm6a677vJxVQAAAKhKaFmoAHv37tXq1audtmVkZDj+d86cOU77rrzyStWtW9fxPDc3V1dffbW2bNkiSbr11ls1aNAg7dixo9h7hoWFqUmTJuXzAgAAAAARFirE6tWrNWbMGI/7UlNT3fYlJCQ4hYXk5GT9+OOPjucfffSRPvrooxLv2bdvX61cufLCiwYAAJWev7+/7Ha77Ha7CgoK5O/PnOVWUFBQILvdLkle/29GNyQAAIAqIizszzVPjh49qoKCAh9Wg9IoKCjQ0aNHHc/P/m/oDbQsVIDRo0eXaWXnxo0byzCM8isIAABAUmRkpNLS0iRJp0+f1unTpxUQwMdBMytqUSgSFRXl1fvz7gAAAKgiQkNDVb9+fR05csSxzfXDKMyrfv36CgkJ8eo9CQsAAABVSFRUlIKDg3Xq1CllZmbSFcnk/P39FRYWpqioKK8HBYmwAAAAUOWEhIT45IMnrIcBzgAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjxjgjGKdPZVacnKyDysBAADAuZz9ea28psQlLKBYx44dczzu3r27DysBAADA+Th27JgaN25c5uvQDQkAAACARzbDMAxfFwFzysnJ0fbt2yVJtWrVYjn4/5ecnOxoaVm3bp3q1avn44pQGfC+QkXgfYWKwPvKvOx2u6NnSLt27cplLQ0+/aFYISEh6tatm6/LMLV69eopNjbW12WgkuF9hYrA+woVgfeV+ZRH16Oz0Q0JAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4xKJsAAAAADyiZQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERZQpaWkpGjJkiV65plndNVVV6lmzZqy2Wyy2WwaPXr0eV9v2bJluvbaaxUbG6vg4GDFxsbq2muv1bJly8q/eJjWhg0b9I9//EODBw92vBfCw8PVokULjRkzRqtXrz6v6/G+wunTpzVv3jxNnDhRffv2VfPmzRUVFaWgoCDVrl1b/fr108svv6zU1NRSXe/HH3/UyJEj1ahRI4WEhKhu3bq64oor9PHHH1fwK4FVPProo46/hzabTStXrjznOfyuqqQMoAqTVOzP7bffXurrFBQUGGPHji3xenfeeadRUFBQcS8GptC7d+8S3wdFP6NGjTJyc3NLvBbvKxT5+uuvS/W+qlmzprF8+fISrzV58mTDz8+v2GtcffXVRnZ2tpdeGcxo8+bNRkBAgNP7IiEhodjj+V1VudGyAPy/hg0bavDgwRd07pNPPqnZs2dLkjp16qSPP/5Y69at08cff6xOnTpJkt5991099dRT5VYvzOnIkSOSpPr16+vBBx/Up59+qnXr1mnt2rV69dVX1aBBA0nSf//733O2XvG+wtni4uI0atQovf766/r888+1du1arVmzRvPnz9cNN9wgf39/HT9+XNdcc422bt3q8RozZszQc889p8LCQjVr1kyzZ8/WunXrtHDhQvXv31+StHTpUt1xxx3efGkwkcLCQt19992y2+2qXbt2qc7hd1Ul5+u0AvjSM888YyxevNg4evSoYRiGceDAgfNuWdizZ4/jG5iuXbsaWVlZTvszMzONrl27GpKMgIAA47fffivvlwETufrqq4358+cbdrvd4/5jx44ZLVq0cLzPVq1a5fE43lc4W3Hvp7P973//c7yvrr32Wrf9qampRlRUlCHJaNiwoXHs2DG3ewwdOrRU3ySj8po2bZohyWjVqpXx+OOPn/P9wO+qyo+wAJzlQsLCvffe6zhn7dq1Ho9Zu3at45i//e1v5VgxrGjx4sWO98P999/v8RjeV7gQLVu2dHRHcvXSSy853i8ff/yxx/MPHTpk+Pv7G5KMIUOGVHS5MJmDBw8a4eHhhiRj5cqVxuTJk88ZFvhdVfnRDQkoA8MwtGjRIklSq1atdOmll3o87tJLL1XLli0lSYsWLZJhGF6rEeZT1N1Dkvbt2+e2n/cVLlRERIQkKScnx23fwoULJUmRkZG67rrrPJ4fGxurQYMGSZK+/fZbpaenV0yhMKX77rtPGRkZuv3229W3b99zHs/vqqqBsACUwYEDBxx91M/1i7Vo/+HDh5WYmFjRpcHEcnNzHY/9/f3d9vO+woXYs2ePtmzZIunMB7ez5eXlad26dZKkyy67TEFBQcVep+g9lZubqw0bNlRMsTCdTz75REuWLFFMTIz+/e9/l+ocfldVDYQFoAx27tzpeOz6x9nV2ft37dpVYTXB/FatWuV43Lp1a7f9vK9QWllZWfrtt9/06quvqm/fvrLb7ZKkCRMmOB3366+/qqCgQBLvKbg7efKkHnzwQUnSSy+9pJo1a5bqPH5XVQ0Bvi4AsLKkpCTH49jY2BKPjYuLczw+dOhQhdUEcyssLNSUKVMcz2+88Ua3Y3hfoSRz5szRmDFjit3/2GOP6ZZbbnHaxnsKJZk0aZKOHj2qnj17auzYsaU+j/dV1UBYAMrg7P684eHhJR4bFhbmeJyRkVFhNcHcpk2b5ugOct1116lLly5ux/C+woXo2LGjZs6cqW7durnt4z2F4vzwww969913FRAQoHfeeUc2m63U5/K+qhrohgSUwdmDCEvqAyxJwcHBjsfZ2dkVVhPMa9WqVXrsscckSbVr19bbb7/t8TjeVyjJ8OHDtX37dm3fvt0xl/21116rLVu26Oabb9aSJUvczuE9BU/y8vJ09913yzAMPfTQQ2rbtu15nc/7qmogLABlEBIS4nicl5dX4rFnD2oNDQ2tsJpgTr/88ouuvfZa2e12hYSEaMGCBcUueMT7CiWJjo5W27Zt1bZtW3Xr1k0jRozQ559/rv/+97/av3+/hg0bpjlz5jidw3sKnrzwwgvavXu3GjZsqMmTJ5/3+byvqgbCAlAGRdMUSuduVs3MzHQ8PldzLSqXAwcOaPDgwTpx4oT8/f01b9489enTp9jjeV/hQtx222264YYbVFhYqPHjxystLc2xj/cUXO3evVsvvviiJOk///mPUzeh0uJ9VTUwZgEog7MHdJ090MuTswd0nT3QC5XbkSNHNGjQIB05ckQ2m03vvfeehg0bVuI5vK9woYYNG6ZPPvlEmZmZWr58uWOgM+8puJo2bZry8vLUtGlTZWVlad68eW7H7Nixw/H4u+++09GjRyVJQ4cOVVhYGO+rKoKwAJRBmzZtHI93795d4rFn7/c0XSYqn+PHj+vyyy/X/v37JZ359m7UqFHnPI/3FS5UrVq1HI8PHjzoeNyiRQv5+/uroKCA9xQk/dktaP/+/br55pvPefw///lPx+MDBw4oLCyM31VVBN2QgDJo0qSJ6tevL8l57nxPvv/+e0lSgwYN1Lhx44ouDT526tQpXXHFFY55yKdMmaL77ruvVOfyvsKFOnz4sOPx2V09goKC1L17d0nS2rVrS+xfXvSeCw4OVteuXSuoUlQG/K6qGggLQBnYbDZHl5Ldu3frp59+8njcTz/95PhWZdiwYec1NR2sJysrS1dffbU2bdokSXryySf16KOPlvp83le4UAsWLHA8bteundO+4cOHS5JOnz6tzz//3OP5SUlJ+uabbyRJAwcOdOqTjsplzpw5MgyjxJ+zBz0nJCQ4thd92Od3VRVhAHA4cOCAIcmQZNx+++2lOmfPnj2Gv7+/Icno2rWrkZWV5bQ/KyvL6Nq1qyHJCAgIMH799dcKqBxmkZubawwePNjxPnrwwQcv6Dq8r3C2+Ph4Izs7u8RjXn31Vcf7rkmTJobdbnfan5qaakRFRRmSjEaNGhnHjx932m+3242hQ4c6rpGQkFDeLwMWM3ny5HO+H/hdVfkxZgFV2urVq7V3717H8+PHjzse79271236wdGjR7tdo0WLFvr73/+uKVOmaMOGDerZs6ceffRRNWvWTPv27dNLL72kzZs3S5L+/ve/66KLLqqQ1wJzuPnmm7VixQpJ0oABAzR27FinQYKugoKC1KJFC7ftvK9wtmeffVYTJ07U9ddfr169eqlZs2YKDw9Xenq6tm/fro8++khr1qyRdOY9NXPmTPn7+ztdIyYmRi+99JLuueceHTx4UJdccomefPJJtWvXTkeOHNFrr72mhIQESWfex/369fP2y4QF8buqCvB1WgF86fbbb3d8a1Kan+IUFBQYd9xxR4nnjh071igoKPDiq4MvnM/7Sf//DW9xeF+hSKNGjUr1foqNjTVWrFhR4rWeeeYZw2azFXuNIUOGnLMVA1VDaVoWDIPfVZUdYxaAcuDn56fZs2dr6dKlGjZsmOrXr6+goCDVr19fw4YN05dffql3331Xfn78Xw6lx/sKRb766iu98soruu6669S+fXvVqVNHAQEBioiIULNmzXT99dcrPj5ee/bs0eWXX17itZ577jmtXr1at9xyi+Li4hQUFKTatWvr8ssv19y5c7V06VKnxbaAc+F3VeVmMwzD8HURAAAAAMyHiAcAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAACo1J599lnZbDbZbDZflwIAlkNYAIAqKDEx0fEBuiw/AIDKjbAAAPCqOXPmOMJGYmKir8up1FauXOn4t165cqWvywFgQQG+LgAA4H0NGjTQ9u3bi93frl07SVLXrl0VHx/vrbIAACZDWACAKigwMFBt27Y953FhYWGlOg4AUDnRDQkAAACAR4QFAMB5Kyws1IcffqghQ4aobt26CgoKUq1atdS/f39Nnz5deXl5bucU9Z8fM2aMY1uTJk3cBk279q3/6aef9NRTT6lfv36Oe0VGRqpNmza69957tXPnzop+uQ7p6el65ZVXNGDAAKdaOnXqpPvvv19r1qwp9txjx47pqaeeUqdOnRQdHa2QkBA1btxYt912m1avXn3Oe3/33Xe6+eab1aRJE4WGhqpatWpq1KiRLr30Uj3yyCP67rvvHMcWDWDv37+/Y1v//v3d/q3nzJlTpn8PAFWAAQCAC0mGJKNv375u+1JTU42ePXs6jvH007p1ayMxMdHpvISEhBLPKfpJSEhwnBMfH3/O4/39/Y233nqr2NcyefJkx7Fl8fXXXxs1a9Y8Zz2efPXVV0ZkZGSJ5913331GQUGBx/MnTJhwzvvWqFHDcfyBAwdK9W8dHx9fpn8TAJUfYxYAAKVWUFCgv/zlL1q7dq0kqW/fvho/fryaNGmiI0eO6L333tPChQu1a9cuDRw4UFu2bFF4eLgkqVu3btq+fbsWLVqkp556SpL01VdfqX79+k73aNKkieOx3W5X9erVNWzYMPXp00cXXXSRwsLCdOTIEW3atElvvPGGjh8/rvHjx6tVq1YaMGBAhbzuhIQEXXXVVbLb7fL399dtt92mYcOGqWHDhsrJydHOnTu1bNkyLV682O3cLVu2aOjQocrLy1NgYKDGjx+va665RmFhYdq8ebOmTJmiAwcO6K233lJYWJheeuklp/OXLFmi1157TZLUvn173XvvvWrdurWioqJ08uRJ/fLLL/rmm2+0bt06xzlFA9jXr1+vO+64Q5L03nvvqVu3bk7Xjo2NLed/KQCVjq/TCgDAfFRMy8Kbb77p2Ddq1CijsLDQ7dwnnnjCccykSZPc9p/dWnDgwIES60hKSjIyMzOL3X/y5Emjffv2hiSjV69eHo8pa8tCdna2Ub9+fUOSUa1aNaeWD1e///6727Zu3bo5WkC++uort/1paWlGmzZtDEmGn5+fsWPHDqf9t912myHJaNSokZGenl7svVNTU922nd2aU1LdAFAcxiwAAErtrbfekiTVqlVLb775pseF2Z577jm1atVKkjRr1izl5uZe8P0aNGigatWqFbs/KipK//jHPyRJq1evVmpq6gXfqzj//e9/deTIEUnSCy+8oH79+hV7bFxcnNPzdevWaf369ZKku+66S4MHD3Y7p3r16po5c6akM2NBpk+f7rT/6NGjkqTOnTs7Wmk8iYmJOfeLAYDzRFgAAJTKkSNHtGvXLknSjTfeqIiICI/HBQQEOAYxnzhxQps2bSq3GjIzM5WYmKhffvlFO3bs0I4dOxQYGOjYv3Xr1nK7V5ElS5ZIOjON7F133XVe537zzTeOx2PHji32uJ49e6p169Zu50hSvXr1JEnff/+99u3bd173B4CyIiwAAEplx44djseXXHJJiceevf/s8y7E8ePH9cQTT6hly5aKiIhQkyZN1LZtW7Vr107t2rXT1Vdf7XRsedu8ebMkqUuXLiW2cnhS9NqDgoLUsWPHEo8t+jf77bffnGaTGjVqlCQpNTVVbdu21YgRIxQfH6+9e/eeVy0AcCEICwCAUklLS3M8rl27donH1q1b1+N552vjxo1q1aqVXnzxRf36668yDKPE47Ozsy/4XsUpCiBF3/Cfj6LXHhMTo4CAkucUKfo3MwxDJ06ccGwfOHCg3nzzTYWGhionJ0fz58/XHXfcoYsuukixsbG65557KqRFBQAkwgIA4AJ4GqtQ3vLy8nTjjTcqNTVVgYGBevjhh7Vq1SolJycrJydHhmHIMAynrjnnChO+UtZ/r/vuu0+JiYmaNm2ahgwZoqioKEnS4cOHNWPGDHXq1MkxwxQAlCfCAgCgVM4eQPvHH3+UeGzRoFzX887Hd999p/3790uSpk+frldeeUV9+vRR3bp1FRwc7DiuLC0XpVGzZk1JUnJy8nmfW/TaU1NTZbfbSzy26N/MZrOpevXqbvtr166tCRMmaOnSpUpLS9PGjRv11FNPKTo6WoZh6Pnnn9eiRYvOu0YAKAlhAQBQKm3btnU8/vnnn0s89uw5/88+Tyr9t+y//PKL4/FNN91U7HEbNmwo1fUuVOfOnR33ycrKOq9zi157Xl6etmzZUuKxRf9mF110kYKCgko81s/PT507d9Y///lPffvtt47tn3zyidNx3mgBAlC5ERYAAKVSv359x4w9n3zyiTIyMjweV1BQoDlz5kg6My1o0YftIiEhIY7HJU2revY38ZmZmR6PKSws1KxZs0pV/4UaOnSoJCkrK8sxxWlpDRo0yPH4vffeK/a4tWvXaufOnW7nlEbnzp0dLRGuA7xL+28NAMUhLAAASu2+++6TJB07dkwPPPCAx2Oee+45xwffu+66y6nLkOQ8ULikqUAvuugix+Oi8OHq8ccfL9epWT0ZOXKkGjRoIEl68skntWrVqmKPTUpKcnrevXt3de3aVdKZNSfObgUocurUKY0bN07SmRaDe++912n//PnzSxy4vWHDBseA6LNXv5ZK/28NAMWxGWYdDQYA8Jmi7it9+/bVypUrHdsLCgrUu3dvrV27VpI0YMAA/e1vf1OTJk2UnJys9957T59//rkkqVmzZtqyZYvbQmLp6emqXbu2cnJy1LlzZ02ZMkWNGjWSn9+Z768aNGig0NBQZWZmqmnTpkpJSZG/v7/uvPNOXXvttapZs6b27t3r+PDds2dPrVmzRpIUHx+v0aNHO93v2Wef1XPPPSfpwgdAJyQkaPDgwbLb7QoICNBtt92m4cOHKzY2Vrm5udq9e7e+/PJLffHFF27f4G/ZskWXXHKJ8vLyFBQUpPvvv19Dhw5VWFiYNm/erClTpjjGZkyaNEkvvfSS0/mNGzfWqVOnNGzYMPXp00ctWrRQWFiYUlNTtXr1av3nP/9RWlqa/P399dNPPznCSZG4uDglJSWpSZMmeu2119SyZUv5+/tLkurUqVPsehkAIEny2drRAADTkmRIMvr27eu2LzU11ejZs6fjGE8/rVu3NhITE4u9/qRJk4o9NyEhwXHc8uXLjZCQkGKP7devn7Fjxw7H8/j4eLd7TZ482bG/LJYvX25Ur169xNdd3D2++uorIzIyssTz7rvvPqOgoMDt3EaNGp3znsHBwR5fu2EYxvTp04s9r7hzAKAI3ZAAAOclJiZG33//vf773//qyiuvVJ06dRQYGKgaNWqoX79+evPNN7VlyxY1atSo2GtMmTJFs2bNUu/evRUTE+P4ptvVFVdcoQ0bNmjkyJGqX7++AgMDVatWLfXt21czZ87Ut99+q7CwsIp6qW617N+/Xy+88IJ69OihGjVqyN/fX5GRkercubMmTJjgNLD7bIMHD9bevXv1xBNPqGPHjoqMjFRwcLAaNmyoW2+9VT/88IPefPNNR+vK2RISEvT666/r+uuvV7t27VSrVi0FBAQoMjJSnTp10iOPPKKdO3e6tagUuffee/XZZ59p8ODBql279jnXewCAs9ENCQAAAIBHtCwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8Oj/ADrf3Kh2gSFUAAAAAElFTkSuQmCC" - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "fig, ax = plt.subplots(1, 1, figsize=(4, 3), dpi=200)\n", - "ax.plot(\n", - " cost_AGP[9:],\n", - " best_seen_AGP[9:],\n", - " label=\"AGP\"\n", - ")\n", - "ax.plot(\n", - " cost_MES[9:],\n", - " best_seen_MES[9:],\n", - " label=\"MES\"\n", - ")\n", - "\n", - "ax.set_title(\"Branin\", fontsize=\"12\")\n", - "ax.set_xlabel(\"Total cost\", fontsize=\"10\")\n", - "ax.set_ylabel(\"Best seen\", fontsize=\"10\")\n", - "ax.tick_params(labelsize=10)\n", - "ax.legend(loc=\"lower right\", fontsize=\"7\", frameon=True, ncol=1)\n", - "plt.tight_layout()" - ], - "metadata": { - "collapsed": false, - "ExecuteTime": { - "end_time": "2023-12-18T11:41:02.192844700Z", - "start_time": "2023-12-18T11:41:01.912139300Z" - } - }, - "id": "e3604083ed32e0eb" - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/website/tutorials.json b/website/tutorials.json index 57e7caabb9..03d73e065d 100644 --- a/website/tutorials.json +++ b/website/tutorials.json @@ -140,10 +140,6 @@ "id": "discrete_multi_fidelity_bo", "title": "Multi-fidelity Bayesian optimization with discrete fidelities using KG" }, - { - "id": "multi_source_bo", - "title": "Multi-Information Source BO with Augmented Gaussian Processes" - }, { "id": "composite_bo_with_hogp", "title": "Composite Bayesian optimization with the High Order Gaussian Process" From 032568a72048aa995e36ea35b8e79b860991b0ab Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Wed, 7 Feb 2024 15:46:50 +0100 Subject: [PATCH 14/20] Fix lint and autosort issues --- test_community/acquisition/test_multi_source.py | 8 +++++--- .../models/test_gp_regression_multisource.py | 12 ++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/test_community/acquisition/test_multi_source.py b/test_community/acquisition/test_multi_source.py index 85a912b276..f169ac5601 100644 --- a/test_community/acquisition/test_multi_source.py +++ b/test_community/acquisition/test_multi_source.py @@ -5,12 +5,14 @@ # LICENSE file in the root directory of this source tree. import torch - -from botorch_community.acquisition.augmented_multisource import AugmentedUpperConfidenceBound from botorch.exceptions import UnsupportedError -from botorch_community.models.gp_regression_multisource import SingleTaskAugmentedGP from botorch.utils.testing import BotorchTestCase, MockModel, MockPosterior +from botorch_community.acquisition.augmented_multisource import ( + AugmentedUpperConfidenceBound, +) +from botorch_community.models.gp_regression_multisource import SingleTaskAugmentedGP + class TestAugmentedUpperConfidenceBound(BotorchTestCase): def _get_mock_agp(self, batch_shape, dtype): diff --git a/test_community/models/test_gp_regression_multisource.py b/test_community/models/test_gp_regression_multisource.py index c232bbc817..50c621f1b1 100644 --- a/test_community/models/test_gp_regression_multisource.py +++ b/test_community/models/test_gp_regression_multisource.py @@ -13,18 +13,18 @@ from botorch import fit_gpytorch_mll from botorch.exceptions import InputDataError, OptimizationWarning from botorch.models import FixedNoiseGP, SingleTaskGP -from botorch_community.models.gp_regression_multisource import ( - _get_reliable_observations, - FixedNoiseAugmentedGP, - get_random_x_for_agp, - SingleTaskAugmentedGP, -) from botorch.models.transforms import Normalize, Standardize from botorch.posteriors import GPyTorchPosterior from botorch.sampling import SobolQMCNormalSampler from botorch.utils import draw_sobol_samples from botorch.utils.test_helpers import get_pvar_expected from botorch.utils.testing import _get_random_data, BotorchTestCase +from botorch_community.models.gp_regression_multisource import ( + _get_reliable_observations, + FixedNoiseAugmentedGP, + get_random_x_for_agp, + SingleTaskAugmentedGP, +) from gpytorch import ExactMarginalLogLikelihood from gpytorch.kernels import MaternKernel, ScaleKernel from gpytorch.likelihoods import FixedNoiseGaussianLikelihood From d246c0220af778439e91a032a25cfdc64bb09809 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Wed, 7 Feb 2024 16:27:53 +0100 Subject: [PATCH 15/20] Incorporate FixedNoiseAugmentedGP in SingleTaskAugmentedGP --- .../models/gp_regression_multisource.py | 165 ++---------------- .../models/test_gp_regression_multisource.py | 97 +--------- 2 files changed, 24 insertions(+), 238 deletions(-) diff --git a/botorch_community/models/gp_regression_multisource.py b/botorch_community/models/gp_regression_multisource.py index cb60438c08..4933963024 100644 --- a/botorch_community/models/gp_regression_multisource.py +++ b/botorch_community/models/gp_regression_multisource.py @@ -133,21 +133,24 @@ def __init__( raise InputDataError("AGP is meant to be used with more than one source.") train_X = [train_X[torch.where(train_S == s)] for s in sources] train_Y = [train_Y[torch.where(train_S == s)] for s in sources] + if train_Yvar is not None: + train_Yvar = [train_Yvar[torch.where(train_S == s)] for s in sources] self.n_true_points = len(train_X[-1]) self.max_n_cheap_points = max([len(points) for points in train_X[:-1]]) # Init and fit a SingleTaskGP for each source self.models = [ self._init_fit_gp( - x[:, :-1], - y, + train_X[s][..., :-1], + train_Y[s], + None if train_Yvar is None else train_Yvar[s], likelihood, covar_module, mean_module, outcome_transform, input_transform, ) - for x, y in zip(train_X, train_Y) + for s in sources ] # Create the training set for the AGP selecting all @@ -171,6 +174,13 @@ def __init__( for s in sources ] ) + if train_Yvar is not None: + train_Yvar = torch.cat( + [ + train_Yvar[s] if s == 0 else train_Yvar[s][reliable_idxs[s - 1]] + for s in sources + ] + ) super().__init__( train_X, @@ -187,6 +197,7 @@ def _init_fit_gp( self, train_X: Tensor, train_Y: Tensor, + train_Yvar: Optional[Tensor] = None, likelihood: Optional[Likelihood] = None, covar_module: Optional[Module] = None, mean_module: Optional[Mean] = None, @@ -198,6 +209,7 @@ def _init_fit_gp( Args: train_X: A `batch_shape x n x d` tensor of training features. train_Y: A `batch_shape x n x m` tensor of training observations. + train_Y: A `batch_shape x n x m` tensor of training observations. likelihood: A likelihood. If omitted, use a standard GaussianLikelihood with inferred noise level. covar_module: The module computing the covariance (Kernel) matrix. @@ -215,155 +227,10 @@ def _init_fit_gp( The fitted Single Task GP and its Marginal Log Likelihood. """ gp = SingleTaskGP( - train_X, - train_Y, - likelihood=likelihood, - covar_module=covar_module, - mean_module=mean_module, - outcome_transform=outcome_transform, - input_transform=input_transform, - ) - mll = ExactMarginalLogLikelihood(gp.likelihood, gp) - fit_gpytorch_mll(mll) - return gp - - -class FixedNoiseAugmentedGP(FixedNoiseGP): - def __init__( - self, - train_X: Tensor, - train_Y: Tensor, - train_Yvar: Tensor, - m: int = 1, - covar_module: Optional[Module] = None, - mean_module: Optional[Mean] = None, - outcome_transform: Optional[OutcomeTransform] = None, - input_transform: Optional[InputTransform] = None, - ) -> None: - """ - Args: - train_X: A `batch_shape x n x (d + 1)` tensor of training features, - where the additional dimension is for the source parameter. - train_Y: A `batch_shape x n x m` tensor of training observations. - train_Yvar: A `batch_shape x n x m` tensor of observed measurement - noise. - m: The moltiplicator factor of the model standard deviation used to select - points from other sources to add to the Augmented GP. - covar_module: The module computing the covariance (Kernel) matrix. - If omitted, use a `MaternKernel`. - mean_module: The mean function to be used. If omitted, use a - `ConstantMean`. - outcome_transform: An outcome transform that is applied to the - training data during instantiation and to the posterior during - inference (that is, the `Posterior` obtained by calling - `.posterior` on the model will be on the original scale). - input_transform: An input transform that is applied in the model's - forward pass. - """ - if m <= 0: - raise InputDataError(f"The value of m must be greater than 0, given m={m}.") - if 0 not in train_X[..., -1]: - raise InputDataError( - "At least one observation of the true source have to be provided." - ) - # Divide train_X and train_Y based on the source - train_S = train_X[..., -1] - sources = torch.unique(train_S).int() - if sources.shape[0] == 1: - raise InputDataError("AGP is meant to be used with more than one source.") - - train_X = [train_X[torch.where(train_S == s)] for s in sources] - train_Y = [train_Y[torch.where(train_S == s)] for s in sources] - train_Yvar = [train_Yvar[torch.where(train_S == s)] for s in sources] - self.n_true_points = len(train_X[-1]) - self.max_n_cheap_points = max([len(points) for points in train_X[:-1]]) - - # Init and fit a SingleTaskGP for each source - self.models = [ - self._init_fit_gp( - x[:, :-1], - y, - yvar, - covar_module, - mean_module, - outcome_transform, - input_transform, - ) - for x, y, yvar in zip(train_X, train_Y, train_Yvar) - ] - - # Create the training set for the AGP selecting all the - # observations from the high fidelity source - # and the reliable observations from the other sources - reliable_idxs = [ - _get_reliable_observations( - self.models[0], self.models[s], train_X[s][:, :-1], m - ) - for s in sources[1:] - ] - train_X = torch.cat( - [ - train_X[s] if s == 0 else train_X[s][reliable_idxs[s - 1]] - for s in sources - ] - )[:, :-1] - train_Y = torch.cat( - [ - train_Y[s] if s == 0 else train_Y[s][reliable_idxs[s - 1]] - for s in sources - ] - ) - train_Yvar = torch.cat( - [ - train_Yvar[s] if s == 0 else train_Yvar[s][reliable_idxs[s - 1]] - for s in sources - ] - ) - - super().__init__( train_X, train_Y, train_Yvar, - covar_module, - mean_module, - outcome_transform, - input_transform, - ) - - def _init_fit_gp( - self, - train_X: Tensor, - train_Y: Tensor, - train_Yvar: Tensor, - covar_module: Optional[Module] = None, - mean_module: Optional[Mean] = None, - outcome_transform: Optional[OutcomeTransform] = None, - input_transform: Optional[InputTransform] = None, - ) -> FixedNoiseGP: - r"""Initialize and fit a Fixed Noise GP model. - - Args: - train_X: A `batch_shape x n x d` tensor of training features. - train_Y: A `batch_shape x n x m` tensor of training observations. - train_Yvar: A `batch_shape x n x m` tensor of observed measurement - noise. - covar_module: The module computing the covariance (Kernel) matrix. - If omitted, use a `MaternKernel`. - mean_module: The mean function to be used. If omitted, use a - `ConstantMean`. - outcome_transform: An outcome transform that is applied to the - training data during instantiation and to the posterior during - inference (that is, the `Posterior` obtained by calling - `.posterior` on the model will be on the original scale). - input_transform: An input transform that is applied in the model's - forward pass. - Returns: - The fitted Fixed Noise GP and its Marginal Log Likelihood. - """ - gp = FixedNoiseGP( - train_X, - train_Y, - train_Yvar=train_Yvar, + likelihood=likelihood, covar_module=covar_module, mean_module=mean_module, outcome_transform=outcome_transform, diff --git a/test_community/models/test_gp_regression_multisource.py b/test_community/models/test_gp_regression_multisource.py index 50c621f1b1..1d77fdc6ef 100644 --- a/test_community/models/test_gp_regression_multisource.py +++ b/test_community/models/test_gp_regression_multisource.py @@ -9,6 +9,7 @@ import warnings import torch +from gpytorch.likelihoods import FixedNoiseGaussianLikelihood from botorch import fit_gpytorch_mll from botorch.exceptions import InputDataError, OptimizationWarning @@ -21,13 +22,11 @@ from botorch.utils.testing import _get_random_data, BotorchTestCase from botorch_community.models.gp_regression_multisource import ( _get_reliable_observations, - FixedNoiseAugmentedGP, get_random_x_for_agp, SingleTaskAugmentedGP, ) from gpytorch import ExactMarginalLogLikelihood from gpytorch.kernels import MaternKernel, ScaleKernel -from gpytorch.likelihoods import FixedNoiseGaussianLikelihood from gpytorch.means import ConstantMean from gpytorch.priors import GammaPrior @@ -51,6 +50,7 @@ def _get_model_and_data( n, d, n_source, + train_Yvar=False, outcome_transform=None, input_transform=None, extra_model_kwargs=None, @@ -63,6 +63,7 @@ def _get_model_and_data( model_kwargs = { "train_X": train_X, "train_Y": train_Y, + "train_Yvar": torch.full_like(train_Y, 0.01) if train_Yvar else None, "outcome_transform": outcome_transform, "input_transform": input_transform, } @@ -126,11 +127,12 @@ def test_get_reliable_observation(self): def test_gp(self): bounds = torch.tensor([[-1.0], [1.0]]) d = 5 - for batch_shape, dtype, use_octf, use_intf in itertools.product( + for batch_shape, dtype, use_octf, use_intf, train_Yvar in itertools.product( (torch.Size(), torch.Size([2])), (torch.float, torch.double), (False, True), (False, True), + (False, True), ): tkwargs = {"device": self.device, "dtype": dtype} octf = Standardize(m=1, batch_shape=torch.Size()) if use_octf else None @@ -144,6 +146,7 @@ def test_gp(self): n=10, d=d, n_source=5, + train_Yvar=train_Yvar, outcome_transform=octf, input_transform=intf, **tkwargs, @@ -325,92 +328,6 @@ def test_condition_on_observations(self): ) ) - -class TestAugmentedFixedNoiseGP(TestAugmentedSingleTaskGP): - def _get_model_and_data( - self, - batch_shape, - n, - d, - n_source, - outcome_transform=None, - input_transform=None, - extra_model_kwargs=None, - **tkwargs, - ): - extra_model_kwargs = extra_model_kwargs or {} - train_X, train_Y = _get_random_data_with_source( - batch_shape, n, d, n_source, **tkwargs - ) - model_kwargs = { - "train_X": train_X, - "train_Y": train_Y, - "train_Yvar": torch.full_like(train_Y, 0.01), - "outcome_transform": outcome_transform, - "input_transform": input_transform, - } - model = FixedNoiseAugmentedGP(**model_kwargs, **extra_model_kwargs) - return model, model_kwargs - - def test_init_error(self): - n, d = 10, 5 - for n_source, batch_shape in itertools.product( - (1, 2, 3), (torch.Size([]), torch.Size([2])) - ): - # Test initialization - train_X, train_Y = _get_random_data_with_source( - batch_shape=batch_shape, n=n, d=d, n_source=n_source - ) - if n_source == 1: - self.assertRaises( - InputDataError, - FixedNoiseAugmentedGP, - train_X, - train_Y, - torch.full_like(train_Y, 0.01), - ) - continue - else: - model = FixedNoiseAugmentedGP( - train_X, train_Y, torch.full_like(train_Y, 0.01) - ) - self.assertIsInstance(model, FixedNoiseAugmentedGP) - - # Test initialization with m = 0 - self.assertRaises( - InputDataError, - FixedNoiseAugmentedGP, - train_X, - train_Y, - torch.full_like(train_Y, 0.01), - m=0, - ) - # Test initialization without true source points - bounds = torch.stack([torch.zeros(d), torch.ones(d)]) - bounds[0, -1] = 1 - bounds[-1, -1] = n_source - 1 - train_X = draw_sobol_samples(bounds=bounds, n=n, q=1).squeeze(1) - train_X[:, -1] = torch.round(train_X[:, -1], decimals=0) - self.assertRaises( - InputDataError, - FixedNoiseAugmentedGP, - train_X, - train_Y, - torch.full_like(train_Y, 0.01), - ) - - def test_get_reliable_observation(self): - x = torch.linspace(0, 5, 15).reshape(-1, 1) - true_y = torch.sin(x).reshape(-1, 1) - y = torch.cos(x).reshape(-1, 1) - - model0 = FixedNoiseGP(x, true_y, torch.full_like(true_y, 1)) - model1 = FixedNoiseGP(x, y, torch.full_like(true_y, 1)) - - res = _get_reliable_observations(model0, model1, x) - true_res = torch.cat([torch.arange(0, 4, 1), torch.arange(10, 13, 1)]).int() - self.assertListEqual(res.tolist(), true_res.tolist()) - def test_fixed_noise_likelihood(self): for batch_shape, dtype in itertools.product( (torch.Size(), torch.Size([2])), (torch.float, torch.double) @@ -421,6 +338,7 @@ def test_fixed_noise_likelihood(self): n=10, d=5, n_source=5, + train_Yvar=True, **tkwargs, ) self.assertIsInstance(model.likelihood, FixedNoiseGaussianLikelihood) @@ -447,6 +365,7 @@ def test_fantasized_noise(self): n=10, d=d, n_source=5, + train_Yvar=True, outcome_transform=octf, **tkwargs, ) From 83c7fa0b5e5ee6b48222d9a84261f0938c944891 Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Wed, 7 Feb 2024 17:55:45 +0100 Subject: [PATCH 16/20] AGP and unit tests refactor --- .../models/gp_regression_multisource.py | 24 +++++++++---------- .../models/test_gp_regression_multisource.py | 12 ++-------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/botorch_community/models/gp_regression_multisource.py b/botorch_community/models/gp_regression_multisource.py index 4933963024..9870e24c5e 100644 --- a/botorch_community/models/gp_regression_multisource.py +++ b/botorch_community/models/gp_regression_multisource.py @@ -26,7 +26,7 @@ from botorch import fit_gpytorch_mll from botorch.exceptions import InputDataError -from botorch.models import FixedNoiseGP, SingleTaskGP +from botorch.models import SingleTaskGP from botorch.models.transforms.input import InputTransform from botorch.models.transforms.outcome import OutcomeTransform from botorch.utils import draw_sobol_samples @@ -45,7 +45,7 @@ def get_random_x_for_agp( ): r"""Draw qMC samples from the box defined by bounds. The function assures that at least one point belong to - the source 0 (the ground truth). + the highest fidelity source (the source associated to bounds[1, -1]). Args: n: The number of samples. @@ -67,9 +67,9 @@ def get_random_x_for_agp( ) train_x = draw_sobol_samples(bounds=bounds, n=n, q=q, seed=seed).squeeze(1) train_x[..., -1] = torch.round(train_x[..., -1], decimals=0) - if 0 not in train_x[..., -1]: + if bounds[1, -1] not in train_x[..., -1]: true_idxs = torch.randint(0, n, [max(1, int(n * 0.2))]) - train_x[true_idxs, -1] = 0 + train_x[true_idxs, -1] = bounds[1, -1] return train_x @@ -122,10 +122,6 @@ def __init__( """ if m <= 0: raise InputDataError(f"The value of m must be greater than 0, given m={m}.") - if 0 not in train_X[..., -1]: - raise InputDataError( - "At least one observation of the true source have to be provided." - ) # Divide train_X and train_Y based on the source train_S = train_X[..., -1] sources = torch.unique(train_S).int() @@ -158,26 +154,28 @@ def __init__( # and the reliable observations from the other sources reliable_idxs = [ _get_reliable_observations( - self.models[0], self.models[s], train_X[s][:, :-1], m + self.models[-1], self.models[s], train_X[s][..., :-1], m ) - for s in sources[1:] + for s in sources[:-1] ] train_X = torch.cat( [ - train_X[s] if s == 0 else train_X[s][reliable_idxs[s - 1]] + train_X[s] if s == len(sources) - 1 else train_X[s][reliable_idxs[s]] for s in sources ] )[:, :-1] train_Y = torch.cat( [ - train_Y[s] if s == 0 else train_Y[s][reliable_idxs[s - 1]] + train_Y[s] if s == len(sources) - 1 else train_Y[s][reliable_idxs[s]] for s in sources ] ) if train_Yvar is not None: train_Yvar = torch.cat( [ - train_Yvar[s] if s == 0 else train_Yvar[s][reliable_idxs[s - 1]] + train_Yvar[s] + if s == len(sources) - 1 + else train_Yvar[s][reliable_idxs[s]] for s in sources ] ) diff --git a/test_community/models/test_gp_regression_multisource.py b/test_community/models/test_gp_regression_multisource.py index 1d77fdc6ef..8cb4929e8c 100644 --- a/test_community/models/test_gp_regression_multisource.py +++ b/test_community/models/test_gp_regression_multisource.py @@ -9,7 +9,6 @@ import warnings import torch -from gpytorch.likelihoods import FixedNoiseGaussianLikelihood from botorch import fit_gpytorch_mll from botorch.exceptions import InputDataError, OptimizationWarning @@ -17,7 +16,6 @@ from botorch.models.transforms import Normalize, Standardize from botorch.posteriors import GPyTorchPosterior from botorch.sampling import SobolQMCNormalSampler -from botorch.utils import draw_sobol_samples from botorch.utils.test_helpers import get_pvar_expected from botorch.utils.testing import _get_random_data, BotorchTestCase from botorch_community.models.gp_regression_multisource import ( @@ -27,6 +25,7 @@ ) from gpytorch import ExactMarginalLogLikelihood from gpytorch.kernels import MaternKernel, ScaleKernel +from gpytorch.likelihoods import FixedNoiseGaussianLikelihood from gpytorch.means import ConstantMean from gpytorch.priors import GammaPrior @@ -79,7 +78,7 @@ def test_data_init(self): self.assertRaises(InputDataError, get_random_x_for_agp, n, bounds, 1) else: x = get_random_x_for_agp(n, bounds, q=1) - self.assertIn(0, x[..., -1]) + self.assertIn(n_source - 1, x[..., -1]) self.assertEqual(x.shape, (n, d)) def test_init_error(self): @@ -104,13 +103,6 @@ def test_init_error(self): self.assertRaises( InputDataError, SingleTaskAugmentedGP, train_X, train_Y, m=0 ) - # Test initialization without true source points - bounds = torch.stack([torch.zeros(d), torch.ones(d)]) - bounds[0, -1] = 1 - bounds[-1, -1] = n_source - 1 - train_X = draw_sobol_samples(bounds=bounds, n=n, q=1).squeeze(1) - train_X[:, -1] = torch.round(train_X[:, -1], decimals=0) - self.assertRaises(InputDataError, SingleTaskAugmentedGP, train_X, train_Y) def test_get_reliable_observation(self): x = torch.linspace(0, 5, 15).reshape(-1, 1) From 8d6a9dc459f6e527d604a118e03a38e735571dac Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Wed, 7 Feb 2024 18:17:37 +0100 Subject: [PATCH 17/20] AUCB and unit tests refactor --- .../acquisition/augmented_multisource.py | 4 ++-- .../acquisition/test_multi_source.py | 22 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/botorch_community/acquisition/augmented_multisource.py b/botorch_community/acquisition/augmented_multisource.py index fdfef202d1..dc01f8a37e 100644 --- a/botorch_community/acquisition/augmented_multisource.py +++ b/botorch_community/acquisition/augmented_multisource.py @@ -99,8 +99,8 @@ def forward(self, X: Tensor) -> Tensor: (agp_mean if self.maximize else -agp_mean) + self.beta.sqrt() * agp_sigma ) source_idxs = { - int(s.tolist()): torch.where(torch.round(X[..., -1], decimals=0) == s)[0] - for s in torch.round(X[..., -1], decimals=0).unique() + s.item(): torch.where(torch.round(X[..., -1], decimals=0) == s)[0] + for s in torch.round(X[..., -1], decimals=0).unique().int() } for s in source_idxs: mean, sigma = self._mean_and_sigma( diff --git a/test_community/acquisition/test_multi_source.py b/test_community/acquisition/test_multi_source.py index f169ac5601..864c9a898d 100644 --- a/test_community/acquisition/test_multi_source.py +++ b/test_community/acquisition/test_multi_source.py @@ -17,7 +17,7 @@ class TestAugmentedUpperConfidenceBound(BotorchTestCase): def _get_mock_agp(self, batch_shape, dtype): train_X = torch.tensor([[0, 0], [0, 1]], dtype=dtype, device=self.device) - train_Y = torch.tensor([[0.5], [5.0]], dtype=dtype, device=self.device) + train_Y = torch.tensor([[5.0], [0.5]], dtype=dtype, device=self.device) rep_shape = batch_shape + torch.Size([1, 1]) train_X = train_X.repeat(rep_shape) train_Y = train_Y.repeat(rep_shape) @@ -34,12 +34,12 @@ def test_upper_confidence_bound(self): module = AugmentedUpperConfidenceBound( model=mm, beta=1.0, - best_f=torch.tensor(0.5, device=self.device, dtype=dtype), - cost={0: 1, 1: 0.5}, + best_f=torch.tensor(5.0, device=self.device, dtype=dtype), + cost={0: 0.5, 1: 1}, ) - X = torch.zeros(1, 2, device=self.device, dtype=dtype) + X = torch.tensor([[0, 1]], device=self.device, dtype=dtype) ucb = module(X) - ucb_expected = torch.tensor([1.8460], device=self.device, dtype=dtype) + ucb_expected = torch.tensor([8.0169], device=self.device, dtype=dtype) self.assertAllClose(ucb, ucb_expected, atol=1e-4) module = AugmentedUpperConfidenceBound( @@ -47,9 +47,9 @@ def test_upper_confidence_bound(self): beta=1.0, maximize=False, best_f=torch.tensor(0.5, device=self.device, dtype=dtype), - cost={0: 1, 1: 0.5}, + cost={0: 0.5, 1: 1}, ) - X = torch.zeros(1, 2, device=self.device, dtype=dtype) + X = torch.tensor([[0, 1]], device=self.device, dtype=dtype) ucb = module(X) ucb_expected = torch.tensor([0.1217], device=self.device, dtype=dtype) self.assertAllClose(ucb, ucb_expected, atol=1e-4) @@ -63,7 +63,7 @@ def test_upper_confidence_bound(self): model=mm1, beta=1.0, best_f=torch.tensor(1.0, device=self.device, dtype=dtype), - cost={0: 1, 1: 0.5}, + cost={0: 0.5, 1: 1.0}, ) # check for proper error if multi-output model mean2 = torch.rand(1, 2, device=self.device, dtype=dtype) @@ -75,7 +75,7 @@ def test_upper_confidence_bound(self): model=mm2, beta=1.0, best_f=torch.tensor(1.0, device=self.device, dtype=dtype), - cost={0: 1, 1: 0.5}, + cost={0: 0.5, 1: 1.0}, ) def test_upper_confidence_bound_batch(self): @@ -85,9 +85,9 @@ def test_upper_confidence_bound_batch(self): model=mm, beta=1.0, best_f=torch.tensor(1.0, device=self.device, dtype=dtype), - cost={0: 1, 1: 0.5}, + cost={0: 0.5, 1: 1.0}, ) - X = torch.zeros(1, 2, device=self.device, dtype=dtype) + X = torch.tensor([[0, 1]], device=self.device, dtype=dtype) ucb = module(X) ucb_expected = torch.tensor([2.3892], device=self.device, dtype=dtype) self.assertAllClose(ucb, ucb_expected, atol=1e-4) From 12e81e7f932daf7046cdcc0caf1db8b7b7c4c3eb Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Thu, 8 Feb 2024 09:06:09 +0100 Subject: [PATCH 18/20] Notebook execution --- notebooks_community/multi_source_bo.ipynb | 290 ++++++++++++---------- 1 file changed, 153 insertions(+), 137 deletions(-) diff --git a/notebooks_community/multi_source_bo.ipynb b/notebooks_community/multi_source_bo.ipynb index 6475812b5d..6210b1ee4d 100644 --- a/notebooks_community/multi_source_bo.ipynb +++ b/notebooks_community/multi_source_bo.ipynb @@ -48,7 +48,7 @@ "output_type": "stream", "text": [ "\n", - "[notice] A new release of pip is available: 23.2.1 -> 23.3.2\n", + "[notice] A new release of pip is available: 23.2.1 -> 24.0\n", "[notice] To update, run: python.exe -m pip install --upgrade pip\n" ] } @@ -59,8 +59,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:04:03.628067200Z", - "start_time": "2024-01-29T08:03:59.110048Z" + "end_time": "2024-02-08T07:54:55.714265100Z", + "start_time": "2024-02-08T07:54:52.594530700Z" } }, "id": "8aa9032dbb2c2b04" @@ -89,8 +89,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:04:09.639408100Z", - "start_time": "2024-01-29T08:04:03.628965400Z" + "end_time": "2024-02-08T07:55:06.365087700Z", + "start_time": "2024-02-08T07:54:55.714265100Z" } }, "id": "e55defd1ee4a5b0f" @@ -102,8 +102,8 @@ "metadata": { "collapsed": true, "ExecuteTime": { - "end_time": "2024-01-29T08:04:09.678340900Z", - "start_time": "2024-01-29T08:04:09.664287600Z" + "end_time": "2024-02-08T07:55:07.689089500Z", + "start_time": "2024-02-08T07:55:06.361808100Z" } }, "outputs": [], @@ -126,8 +126,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:04:09.679342100Z", - "start_time": "2024-01-29T08:04:09.669295Z" + "end_time": "2024-02-08T07:55:07.701088200Z", + "start_time": "2024-02-08T07:55:07.690705Z" } }, "id": "e316bd291459a135" @@ -165,8 +165,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:04:09.761367100Z", - "start_time": "2024-01-29T08:04:09.677340600Z" + "end_time": "2024-02-08T07:55:07.793085900Z", + "start_time": "2024-02-08T07:55:07.698958300Z" } }, "id": "5f13380e681011ea" @@ -214,8 +214,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:04:09.772956200Z", - "start_time": "2024-01-29T08:04:09.765101800Z" + "end_time": "2024-02-08T07:55:07.801808200Z", + "start_time": "2024-02-08T07:55:07.797392200Z" } }, "id": "f8272160f69227ef" @@ -262,8 +262,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:04:09.783470800Z", - "start_time": "2024-01-29T08:04:09.769617200Z" + "end_time": "2024-02-08T07:55:07.811037100Z", + "start_time": "2024-02-08T07:55:07.802897300Z" } }, "id": "311309bd6a4d3a92" @@ -290,8 +290,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:04:11.943091700Z", - "start_time": "2024-01-29T08:04:09.778471900Z" + "end_time": "2024-02-08T07:55:10.200578200Z", + "start_time": "2024-02-08T07:55:07.808815500Z" } }, "id": "a2b9dbfbae2f7d5" @@ -314,56 +314,72 @@ "name": "stdout", "output_type": "stream", "text": [ - "Iter 0;\t Fid = 1.00;\t Obj = -12.0999;\n", - "Iter 1;\t Fid = 1.00;\t Obj = -50.0743;\n", - "Iter 2;\t Fid = 0.50;\t Obj = -11.9672;\n", - "Iter 3;\t Fid = 0.50;\t Obj = -8.7046;\n", - "Iter 4;\t Fid = 0.50;\t Obj = -13.3425;\n", - "Iter 5;\t Fid = 1.00;\t Obj = -11.6850;\n", - "Iter 6;\t Fid = 0.50;\t Obj = -8.2926;\n", - "Iter 7;\t Fid = 1.00;\t Obj = -14.4455;\n", - "Iter 8;\t Fid = 0.50;\t Obj = -16.5711;\n", - "Iter 9;\t Fid = 1.00;\t Obj = -32.7208;\n", - "Iter 10;\t Fid = 0.50;\t Obj = -12.7833;\n", - "Iter 11;\t Fid = 1.00;\t Obj = -40.2770;\n", - "Iter 12;\t Fid = 0.50;\t Obj = -4.8935;\n", - "Iter 13;\t Fid = 1.00;\t Obj = -15.2650;\n", - "Iter 14;\t Fid = 0.50;\t Obj = -6.9511;\n", - "Iter 15;\t Fid = 1.00;\t Obj = -18.1231;\n", - "Iter 16;\t Fid = 0.50;\t Obj = -0.6015;\n", - "Iter 17;\t Fid = 1.00;\t Obj = -17.0770;\n", - "Iter 18;\t Fid = 0.50;\t Obj = -4.3991;\n", - "Iter 19;\t Fid = 1.00;\t Obj = -12.4392;\n", - "Iter 20;\t Fid = 1.00;\t Obj = -112.9747;\n", - "Iter 21;\t Fid = 0.50;\t Obj = -4.6979;\n", - "Iter 22;\t Fid = 0.50;\t Obj = -13.1965;\n", - "Iter 23;\t Fid = 1.00;\t Obj = -14.2637;\n", - "Iter 24;\t Fid = 0.50;\t Obj = -16.4335;\n", - "Iter 25;\t Fid = 1.00;\t Obj = -1.4522;\n", - "Iter 26;\t Fid = 0.50;\t Obj = -14.7771;\n", - "Iter 27;\t Fid = 1.00;\t Obj = -14.4608;\n", - "Iter 28;\t Fid = 1.00;\t Obj = -15.3493;\n", - "Iter 29;\t Fid = 1.00;\t Obj = -27.3031;\n", - "Iter 30;\t Fid = 0.50;\t Obj = -13.6875;\n", - "Iter 31;\t Fid = 0.50;\t Obj = -9.9730;\n", - "Iter 32;\t Fid = 0.50;\t Obj = -9.1073;\n", - "Iter 33;\t Fid = 1.00;\t Obj = -20.4122;\n", - "Iter 34;\t Fid = 1.00;\t Obj = -11.3355;\n", - "Iter 35;\t Fid = 0.50;\t Obj = -31.8170;\n", - "Iter 36;\t Fid = 0.50;\t Obj = -9.0516;\n", - "Iter 37;\t Fid = 1.00;\t Obj = -6.1756;\n", - "Iter 38;\t Fid = 0.50;\t Obj = -4.9320;\n", - "Iter 39;\t Fid = 1.00;\t Obj = -7.7986;\n", - "Iter 40;\t Fid = 0.50;\t Obj = -8.6059;\n", - "Iter 41;\t Fid = 1.00;\t Obj = -17.1693;\n", - "Iter 42;\t Fid = 0.50;\t Obj = -10.7690;\n", - "Iter 43;\t Fid = 1.00;\t Obj = -25.7958;\n", - "Iter 44;\t Fid = 1.00;\t Obj = -122.8804;\n", - "Iter 45;\t Fid = 1.00;\t Obj = -8.2496;\n", - "Iter 46;\t Fid = 0.50;\t Obj = -0.7404;\n", - "Iter 47;\t Fid = 1.00;\t Obj = -13.2155;\n", - "Iter 48;\t Fid = 1.00;\t Obj = -0.4122;\n", - "Iter 49;\t Fid = 0.50;\t Obj = -0.4802;\n" + "Iter 0;\t Fid = 1.00;\t Obj = -200.3252;\n", + "Iter 1;\t Fid = 1.00;\t Obj = -38.1094;\n", + "Iter 2;\t Fid = 1.00;\t Obj = -38.1090;\n", + "Iter 3;\t Fid = 1.00;\t Obj = -38.1093;\n", + "Iter 4;\t Fid = 1.00;\t Obj = -38.1093;\n", + "Iter 5;\t Fid = 0.75;\t Obj = -10.3835;\n", + "Iter 6;\t Fid = 1.00;\t Obj = -38.1093;\n", + "Iter 7;\t Fid = 1.00;\t Obj = -36.9691;\n", + "Iter 8;\t Fid = 1.00;\t Obj = -38.0601;\n", + "Iter 9;\t Fid = 1.00;\t Obj = -36.4893;\n", + "Iter 10;\t Fid = 1.00;\t Obj = -34.5676;\n", + "Iter 11;\t Fid = 1.00;\t Obj = -27.2647;\n", + "Iter 12;\t Fid = 1.00;\t Obj = -23.7780;\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\ponti\\Desktop\\workspace\\botorch\\botorch\\optim\\optimize.py:367: RuntimeWarning: Optimization failed in `gen_candidates_scipy` with the following warning(s):\n", + "[OptimizationWarning('Optimization failed within `scipy.optimize.minimize` with status 2 and message ABNORMAL_TERMINATION_IN_LNSRCH.')]\n", + "Trying again with a new set of initial conditions.\n", + " warnings.warn(first_warn_msg, RuntimeWarning)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iter 13;\t Fid = 1.00;\t Obj = -50.3585;\n", + "Iter 14;\t Fid = 1.00;\t Obj = -35.5112;\n", + "Iter 15;\t Fid = 1.00;\t Obj = -17.9104;\n", + "Iter 16;\t Fid = 1.00;\t Obj = -12.9752;\n", + "Iter 17;\t Fid = 1.00;\t Obj = -13.3390;\n", + "Iter 18;\t Fid = 1.00;\t Obj = -8.9100;\n", + "Iter 19;\t Fid = 1.00;\t Obj = -3.2656;\n", + "Iter 20;\t Fid = 1.00;\t Obj = -2.7450;\n", + "Iter 21;\t Fid = 1.00;\t Obj = -5.7930;\n", + "Iter 22;\t Fid = 1.00;\t Obj = -0.6055;\n", + "Iter 23;\t Fid = 1.00;\t Obj = -0.5995;\n", + "Iter 24;\t Fid = 1.00;\t Obj = -2.8311;\n", + "Iter 25;\t Fid = 1.00;\t Obj = -1.0225;\n", + "Iter 26;\t Fid = 1.00;\t Obj = -2.8477;\n", + "Iter 27;\t Fid = 1.00;\t Obj = -1.0333;\n", + "Iter 28;\t Fid = 1.00;\t Obj = -0.4881;\n", + "Iter 29;\t Fid = 1.00;\t Obj = -3.4325;\n", + "Iter 30;\t Fid = 1.00;\t Obj = -0.8336;\n", + "Iter 31;\t Fid = 1.00;\t Obj = -0.4630;\n", + "Iter 32;\t Fid = 1.00;\t Obj = -2.3562;\n", + "Iter 33;\t Fid = 1.00;\t Obj = -4.9150;\n", + "Iter 34;\t Fid = 1.00;\t Obj = -0.4038;\n", + "Iter 35;\t Fid = 1.00;\t Obj = -1.9965;\n", + "Iter 36;\t Fid = 1.00;\t Obj = -0.5458;\n", + "Iter 37;\t Fid = 1.00;\t Obj = -5.4650;\n", + "Iter 38;\t Fid = 1.00;\t Obj = -0.8234;\n", + "Iter 39;\t Fid = 1.00;\t Obj = -0.6905;\n", + "Iter 40;\t Fid = 1.00;\t Obj = -0.8766;\n", + "Iter 41;\t Fid = 1.00;\t Obj = -0.9116;\n", + "Iter 42;\t Fid = 1.00;\t Obj = -3.7283;\n", + "Iter 43;\t Fid = 1.00;\t Obj = -1.8566;\n", + "Iter 44;\t Fid = 1.00;\t Obj = -1.5902;\n", + "Iter 45;\t Fid = 1.00;\t Obj = -1.2975;\n", + "Iter 46;\t Fid = 1.00;\t Obj = -1.7442;\n", + "Iter 47;\t Fid = 1.00;\t Obj = -6.0570;\n", + "Iter 48;\t Fid = 1.00;\t Obj = -2.5479;\n", + "Iter 49;\t Fid = 1.00;\t Obj = -1.4998;\n" ] } ], @@ -399,8 +415,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:08:22.793747600Z", - "start_time": "2024-01-29T08:04:11.940091600Z" + "end_time": "2024-02-08T07:59:16.933226800Z", + "start_time": "2024-02-08T07:55:10.205902900Z" } }, "id": "8d02e319798a28d7" @@ -433,8 +449,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:08:22.804418600Z", - "start_time": "2024-01-29T08:08:22.796748900Z" + "end_time": "2024-02-08T07:59:16.933226800Z", + "start_time": "2024-02-08T07:59:16.930006100Z" } }, "id": "1c93f20bdaffccac" @@ -462,8 +478,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:08:22.837919300Z", - "start_time": "2024-01-29T08:08:22.806683100Z" + "end_time": "2024-02-08T07:59:16.943971Z", + "start_time": "2024-02-08T07:59:16.935329500Z" } }, "id": "1a1bba8e3496b54d" @@ -480,8 +496,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:08:22.840918900Z", - "start_time": "2024-01-29T08:08:22.819913500Z" + "end_time": "2024-02-08T07:59:16.963122100Z", + "start_time": "2024-02-08T07:59:16.943971Z" } }, "id": "e8414dfbd0643afa" @@ -494,56 +510,56 @@ "name": "stdout", "output_type": "stream", "text": [ - "Iter 0;\t Fid = 0.50;\t Obj = -8.3276;\n", - "Iter 1;\t Fid = 0.50;\t Obj = -21.1510;\n", - "Iter 2;\t Fid = 0.50;\t Obj = -20.7855;\n", - "Iter 3;\t Fid = 0.50;\t Obj = -8.5067;\n", - "Iter 4;\t Fid = 0.50;\t Obj = -49.7241;\n", - "Iter 5;\t Fid = 0.50;\t Obj = -1.3525;\n", - "Iter 6;\t Fid = 0.50;\t Obj = -98.0777;\n", - "Iter 7;\t Fid = 0.50;\t Obj = -88.9142;\n", - "Iter 8;\t Fid = 0.50;\t Obj = -1.3561;\n", - "Iter 9;\t Fid = 1.00;\t Obj = -135.8402;\n", - "Iter 10;\t Fid = 0.50;\t Obj = -16.4201;\n", - "Iter 11;\t Fid = 0.50;\t Obj = -125.3250;\n", - "Iter 12;\t Fid = 0.50;\t Obj = -6.7582;\n", - "Iter 13;\t Fid = 0.50;\t Obj = -61.3218;\n", - "Iter 14;\t Fid = 0.50;\t Obj = -19.6356;\n", - "Iter 15;\t Fid = 0.50;\t Obj = -91.3633;\n", - "Iter 16;\t Fid = 0.50;\t Obj = -90.9424;\n", - "Iter 17;\t Fid = 0.50;\t Obj = -22.3466;\n", - "Iter 18;\t Fid = 0.50;\t Obj = -1.9025;\n", - "Iter 19;\t Fid = 0.50;\t Obj = -16.0446;\n", - "Iter 20;\t Fid = 0.50;\t Obj = -13.2516;\n", - "Iter 21;\t Fid = 0.50;\t Obj = -23.5304;\n", - "Iter 22;\t Fid = 0.50;\t Obj = -8.6583;\n", - "Iter 23;\t Fid = 0.50;\t Obj = -19.9776;\n", - "Iter 24;\t Fid = 0.50;\t Obj = -22.8034;\n", - "Iter 25;\t Fid = 0.50;\t Obj = -11.0479;\n", - "Iter 26;\t Fid = 0.50;\t Obj = -1.0324;\n", - "Iter 27;\t Fid = 0.50;\t Obj = -3.1515;\n", - "Iter 28;\t Fid = 0.50;\t Obj = -20.1842;\n", - "Iter 29;\t Fid = 0.50;\t Obj = -4.3885;\n", - "Iter 30;\t Fid = 0.50;\t Obj = -262.8294;\n", - "Iter 31;\t Fid = 0.50;\t Obj = -13.1549;\n", - "Iter 32;\t Fid = 0.50;\t Obj = -35.3697;\n", - "Iter 33;\t Fid = 0.50;\t Obj = -9.8748;\n", - "Iter 34;\t Fid = 0.50;\t Obj = -5.5868;\n", - "Iter 35;\t Fid = 0.50;\t Obj = -262.2982;\n", - "Iter 36;\t Fid = 1.00;\t Obj = -1.1024;\n", - "Iter 37;\t Fid = 0.50;\t Obj = -1.9896;\n", - "Iter 38;\t Fid = 1.00;\t Obj = -10.5575;\n", - "Iter 39;\t Fid = 1.00;\t Obj = -8.7057;\n", - "Iter 40;\t Fid = 1.00;\t Obj = -2.3179;\n", - "Iter 41;\t Fid = 1.00;\t Obj = -1.9847;\n", - "Iter 42;\t Fid = 1.00;\t Obj = -6.1164;\n", - "Iter 43;\t Fid = 1.00;\t Obj = -6.2065;\n", - "Iter 44;\t Fid = 0.50;\t Obj = -110.4581;\n", - "Iter 45;\t Fid = 1.00;\t Obj = -2.2320;\n", - "Iter 46;\t Fid = 0.50;\t Obj = -256.6816;\n", - "Iter 47;\t Fid = 1.00;\t Obj = -7.6145;\n", - "Iter 48;\t Fid = 0.75;\t Obj = -15.1647;\n", - "Iter 49;\t Fid = 1.00;\t Obj = -4.1053;\n" + "Iter 0;\t Fid = 0.50;\t Obj = -85.5566;\n", + "Iter 1;\t Fid = 0.50;\t Obj = -7.7401;\n", + "Iter 2;\t Fid = 0.50;\t Obj = -85.9927;\n", + "Iter 3;\t Fid = 0.50;\t Obj = -10.4996;\n", + "Iter 4;\t Fid = 0.50;\t Obj = -90.1834;\n", + "Iter 5;\t Fid = 0.50;\t Obj = -15.9023;\n", + "Iter 6;\t Fid = 0.50;\t Obj = -121.0723;\n", + "Iter 7;\t Fid = 0.50;\t Obj = -96.2876;\n", + "Iter 8;\t Fid = 0.50;\t Obj = -8.3718;\n", + "Iter 9;\t Fid = 0.50;\t Obj = -11.9013;\n", + "Iter 10;\t Fid = 0.50;\t Obj = -15.0995;\n", + "Iter 11;\t Fid = 0.50;\t Obj = -7.6864;\n", + "Iter 12;\t Fid = 0.50;\t Obj = -6.4228;\n", + "Iter 13;\t Fid = 0.50;\t Obj = -51.4467;\n", + "Iter 14;\t Fid = 0.50;\t Obj = -22.0845;\n", + "Iter 15;\t Fid = 0.50;\t Obj = -5.9313;\n", + "Iter 16;\t Fid = 0.50;\t Obj = -130.6765;\n", + "Iter 17;\t Fid = 0.50;\t Obj = -181.9663;\n", + "Iter 18;\t Fid = 0.50;\t Obj = -88.9639;\n", + "Iter 19;\t Fid = 0.50;\t Obj = -21.3274;\n", + "Iter 20;\t Fid = 0.50;\t Obj = -8.2452;\n", + "Iter 21;\t Fid = 0.50;\t Obj = -14.1170;\n", + "Iter 22;\t Fid = 0.50;\t Obj = -8.3733;\n", + "Iter 23;\t Fid = 0.50;\t Obj = -33.3266;\n", + "Iter 24;\t Fid = 0.50;\t Obj = -22.8821;\n", + "Iter 25;\t Fid = 0.50;\t Obj = -1.2653;\n", + "Iter 26;\t Fid = 0.50;\t Obj = -6.2855;\n", + "Iter 27;\t Fid = 0.50;\t Obj = -129.2954;\n", + "Iter 28;\t Fid = 0.50;\t Obj = -27.6052;\n", + "Iter 29;\t Fid = 0.50;\t Obj = -41.9348;\n", + "Iter 30;\t Fid = 0.50;\t Obj = -22.8847;\n", + "Iter 31;\t Fid = 0.50;\t Obj = -2.2557;\n", + "Iter 32;\t Fid = 0.50;\t Obj = -13.0811;\n", + "Iter 33;\t Fid = 0.50;\t Obj = -9.5395;\n", + "Iter 34;\t Fid = 0.50;\t Obj = -13.6012;\n", + "Iter 35;\t Fid = 0.50;\t Obj = -85.0653;\n", + "Iter 36;\t Fid = 0.50;\t Obj = -79.1870;\n", + "Iter 37;\t Fid = 0.50;\t Obj = -7.1699;\n", + "Iter 38;\t Fid = 0.50;\t Obj = -8.6771;\n", + "Iter 39;\t Fid = 1.00;\t Obj = -1.5034;\n", + "Iter 40;\t Fid = 0.50;\t Obj = -122.4489;\n", + "Iter 41;\t Fid = 0.50;\t Obj = -24.0085;\n", + "Iter 42;\t Fid = 0.50;\t Obj = -2.1577;\n", + "Iter 43;\t Fid = 1.00;\t Obj = -4.0370;\n", + "Iter 44;\t Fid = 0.50;\t Obj = -13.9348;\n", + "Iter 45;\t Fid = 1.00;\t Obj = -5.6706;\n", + "Iter 46;\t Fid = 1.00;\t Obj = -3.8358;\n", + "Iter 47;\t Fid = 1.00;\t Obj = -1.0238;\n", + "Iter 48;\t Fid = 0.50;\t Obj = -289.3283;\n", + "Iter 49;\t Fid = 1.00;\t Obj = -3.0379;\n" ] } ], @@ -576,8 +592,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:15:52.142811900Z", - "start_time": "2024-01-29T08:08:22.829920500Z" + "end_time": "2024-02-08T08:03:49.181588500Z", + "start_time": "2024-02-08T07:59:16.951110100Z" } }, "id": "fcaf20bf41f1b680" @@ -604,8 +620,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:15:52.144810Z", - "start_time": "2024-01-29T08:15:52.137410200Z" + "end_time": "2024-02-08T08:03:49.194444700Z", + "start_time": "2024-02-08T08:03:49.188327300Z" } }, "id": "5c5cc1ef1808ad1f" @@ -621,8 +637,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:15:52.159128900Z", - "start_time": "2024-01-29T08:15:52.145813Z" + "end_time": "2024-02-08T08:03:49.200521400Z", + "start_time": "2024-02-08T08:03:49.194444700Z" } }, "id": "4a7f7020f195a31f" @@ -638,20 +654,20 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:15:52.162132Z", - "start_time": "2024-01-29T08:15:52.155612900Z" + "end_time": "2024-02-08T08:03:49.206250200Z", + "start_time": "2024-02-08T08:03:49.200521400Z" } }, "id": "f696548b9b3f50a5" }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 17, "outputs": [ { "data": { "text/plain": "
", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwsAAAJECAYAAABZ37i3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAB7CAAAewgFu0HU+AABs9klEQVR4nO3dd3xUVf7/8fekk0BCr6GEJqCoVF16ExUXAV0VERBExYKKDeuK7n5VlFWsKCIkNqQoCwKCWAIK0psgRakSCQZCTWeS+/uDX2aZkgLJzNy583o+Hjx2cusZnL3kPed8zrEZhmEIAAAAAFyE+LsBAAAAAMyJsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAB4WY8ePWSz2WSz2bRs2TJ/NwcASo2wAAA4L+f+4uvpT0hIiCpVqqSEhAQNHDhQU6dO1enTp/3dbADABSAsAADKlWEYysjI0P79+zV//nzdfffdatasmb766it/Nw0AcJ7C/N0AAEDg6tChgzp27Oi0raCgQCdOnNCWLVu0fft2SdJff/2lG264QfPmzdPf//53fzQVAHABCAsAgAvWr18/Pf/880XuX7lypQYPHqyUlBTl5+frnnvu0b59+xQeHu67RpoAdQoAAhXDkAAAXtO5c2fNmTPH8fOff/7JL84AEEAICwAAr7ryyiuVkJDg+LlwaBIAwPwICwAAr6tTp47jdWZmptv+/fv3O2ZTatSokWP7ihUrdOedd6pFixaKi4uTzWbT2LFjnc4tKCjQTz/9pOeee059+/ZVgwYNFB0drcjISNWpU0e9evXSiy++qKNHj5aqrefO7FRo165dGjt2rFq2bKmKFSsqNjZWl112mZ566qlSXbc0U6eOGDHCcUxSUpIkKSsrS5MnT1aXLl1Uq1YtRUZGqn79+rr11lu1cuXKUr0fACgLahYAAF53+PBhx+vatWuXeHxeXp4efPBBTZkypdjjzpw5o4SEBP35559F3vfw4cNKTk7Wyy+/rPfff19Dhw49r7a///77Gjt2rHJzc522//LLL/rll180depULVmyRO3btz+v65Zk+/bt+sc//qEdO3Y4bU9JSdHMmTM1c+ZMPffcc3rhhRfK9b4AcC7CAgDAq9avX6+9e/c6fu7atWuJ5zz88MOOoNC6dWtddtllCg8P12+//aaQkP91iufn5zuCQsWKFXXxxRercePGio2N1ZkzZ5SSkqLVq1fr1KlTyszM1LBhwxQeHq5bbrmlVG1PSkrSvffeK0m66KKL1L59e1WoUEE7d+7UypUrZRiG0tPTdf3112vHjh2Ki4sr9d9LcQ4dOqQ+ffooNTVVlStXVteuXVW7dm0dPXpUP/zwg06ePClJ+te//qVWrVqV+v0AwHkzAAA4D927dzckGZKM8ePHF3vs2rVrjUaNGjmOHzRokMfj9u3b5zgmNDTUkGTUr1/f+PHHH92OzcnJcbzOzc01Ro4caSQnJxt5eXker52Tk2O8+uqrRlhYmCHJqFy5snH69Oki21zYDklGZGSkUaNGDWPx4sVuxy1fvtyIjY11HPvCCy8Uec1z/86Sk5M9HnP77bc73VeS8cQTTxiZmZlOx6Wnpxu9evVyHNu4cWOjoKCgyHsDQFnQswAAuGBff/2125j9goICnTx5Ur/88ou2bdvm2D5o0CB9+umnJV4zPz9f0dHR+u6779S8eXO3/ZGRkY7XERERmj59erHXi4yM1OOPP66CggI9+eSTOnHihD755BNHj0FJvvvuO1166aVu27t166aXXnpJY8aMkSR9/vnneu6550p1zZLk5ubqqaee0ksvveS2r2rVqpoxY4aaNGmizMxM7d27V2vXrtUVV1xRLvcGgHMRFgAAF2zdunVat25dscfUqVNHkydP1sCBA0t93TFjxngMCmUxcuRIPfnkk5LOBoDShIW7777bY1AoNHz4cI0dO1Z2u127du3SqVOnFBsbW+a21qhRo9jgUatWLV133XWaPXu2JBEWAHgNYQEA4FWpqam68cYbNWTIEL311luqUqVKiecMHjz4vO9TUFCgDRs2aPPmzUpJSdGpU6d05swZj8du3ry5VNe86aabit1fqVIlNWnSRLt27ZJhGDpw4IBat259vk13079/f0VFRRV7TJs2bRxhYf/+/WW+JwB4QlgAAFyw8ePHe1zBOTMzU/v379fixYv16quv6siRI/r000+1adMm/fTTT8UGhvDw8PP6hdtut+utt97SpEmTlJKSUqpzSjuNamnaUa1aNcfrU6dOleq6Zr0vALhinQUAQLmLiYnRxRdfrMcee0ybNm1SvXr1JEm//vqrHnnkkWLPrVKlisLCSvddVm5urq677jo9+uijpQ4KknT69OlSHVea2Y3Cw8Mdr4vqyThf/rovALgiLAAAvKpevXoaP3684+dPP/3Uad0FVxUqVCj1tV944QUtXbpU0tnF1G655RbNnj1bO3bs0MmTJ5WXlyfDMBx/Cp37ujjnLszmS/66LwC4YhgSAMDrrr76asdru92u5cuXl3ltgNzcXL399tuOn5OSkjR8+PAijy9tbwIA4H/oWQAAeF2dOnWcfj5w4ECZr7l27VplZGRIki6++OJig0J53RMAgg1hAQDgdVlZWU4/n7sK84U6dOiQ43VpCoJ//PHHMt8TAIINYQEA4HUbN250+rmw4Lkszg0crmHEVUFBgT744IMy3xMAgg1hAQDgdZMmTXK8ttls6tWrV5mv2bhxY8fr5cuX6+TJk0UeO3HiRG3ZsqXM9wSAYENYAAB4zYkTJzR69GgtWLDAsW3IkCGqVatWma/dpk0bRw/FyZMnddNNNzkNTZLOFkE/99xzevLJJxUTE1PmewJAsGE2JADABfv66689LnCWlZWl/fv3a/Xq1crOznZsb968uV5//fVyuXdISIj+/e9/64477pAkffvtt2revLk6deqkhg0bKj09XcuWLdPx48clSR988IFuu+22crk3AAQLwgIA4IKtW7dO69atK9Wx119/vaZMmaKaNWuW2/1Hjhyp3bt366WXXpJ0duXob7/91umYqKgovfHGGxoyZAhhAQDOE2EBAFDuIiMjFRcXp6ZNm+rKK6/UkCFD1K5dO6/c68UXX9S1116rd955RytWrNCRI0dUqVIlxcfH65prrtGoUaPUrFkzr9wbAKzOZpR2GUsAAAAAQYUCZwAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEhQBw4cECPPvqoWrRooZiYGFWtWlUdOnTQxIkTlZWV5e/mAQAAwIJshmEY/m4EirdgwQINHTpUp06d8ri/efPmWrRokZo2berjlgEAAMDKCAsmt2nTJnXu3FnZ2dmqWLGinnrqKfXs2VPZ2dmaOXOmpk6dKulsYFi/fr0qVark5xYDAADAKggLJtetWzf99NNPCgsL048//qi//e1vTvsnTpyocePGSZLGjx+v559/vtzunZOTo61bt0qSatSoobCwsHK7NgAAAMqX3W7XkSNHJEmtW7dWVFRUma9JWDCxtWvX6oorrpAkjR49Wu+//77bMQUFBbrkkku0Y8cOVa5cWWlpaQoPDy+X+69bt04dO3Ysl2sBAADAd9auXasOHTqU+ToUOJvYvHnzHK9Hjhzp8ZiQkBANHz5cknTixAklJyf7omkAAAAIAowrMbEVK1ZIkmJiYtSuXbsij+vevbvj9cqVK9W3b99yuX+NGjUcr9euXas6deqUy3UBAABQ/lJTUx2jQs79Pa4sCAsmtmPHDklS06ZNi60XaNGihds55eHce9apU0fx8fHldm0AAAB4T3nVmhIWTConJ0dHjx6VpBJ/Sa9SpYpiYmKUmZmpgwcPlvoeKSkpxe5PTU0t9bUAAABgPYQFkzp9+rTjdcWKFUs8vjAsZGRklPoe9evXv6C2AQAAIDhQ4GxSOTk5jtcRERElHh8ZGSlJys7O9lqbAAAAEFzoWTCpc+fFzcvLK/H43NxcSVKFChVKfY+ShiydWyQDAACA4ENYMKlzV2IuzdCizMxMSaUbslSIgmUAAAAUh2FIJhUVFaVq1apJKrkQ+fjx446wQB0CAAAAygthwcRatWolSdq9e7fsdnuRx+3cudPxumXLll5vFwAAAIIDYcHEunTpIunsEKMNGzYUedzy5csdrzt37uz1dgEAACA4EBZMbODAgY7XiYmJHo8pKCjQxx9/LEmqXLmyevbs6YumAQAAIAgQFkysY8eO6tq1qyRp2rRpWrVqldsxr732mmPV5oceekjh4eE+bSMAAACsi9mQTO7NN99U586dlZ2drb59++rpp59Wz549lZ2drZkzZ+qDDz6QJDVv3lyPPvqon1sLAAAAKyEsmFybNm00a9YsDR06VKdOndLTTz/tdkzz5s21aNEip+lWAQAAgLJiGFIA6N+/v3755Rc9/PDDat68uaKjo1W5cmW1b99er7zyijZt2qSmTZv6u5kAAACwGJthGIa/GwFzSklJcazbcPDgQRZxAwAA+P8Mw9Bna/7Qt9v/Uq49v8Tjm9eqpH8NuMSrbfLG724MQwIAAADO08x1B/XsvG2lPj7PXuDF1ngPw5AAAACA85BzJl9vfPebv5vhE4QFAAAA4DzMWX9Qf53K9XczfIJhSAAAAEAp5drzNXnZHqdtl8XH6Ya2xdcH1KgU6c1meQ1hAQAAACilLzakKPVkjtO2p/u11BWNq/mpRd7FMCQAAACgFPLsBZqc7NyrcEVCVcsGBYmwAAAAAJTK3I0p+vNEttO2h3o381NrfIOwAAAAAJTgTH6B3kne7bStQ6Mq+lsT6/YqSIQFAAAAoET/3fSnUo479yo82LuZbDabn1rkG4QFAAAAoBj2/AK969Kr0LZBZXVpWt1PLfIdwgIAAABQjPmbD+lAepbTtmDoVZAICwAAAECR8gsMt1qFy+pXVvfmNfzUIt8iLAAAAABFWLDlkPYdzXTa9lDvpkHRqyARFgAAAACP8gsMvf3D707bWteLU8+LavqpRb5HWAAAAAA8WLQ1VXuOOPcqBEutQiHCAgAAAOCioMDQ29879yq0qhOrPi2Dp1dBIiwAAAAAbhZvO6zf0zKctgVbr4JEWAAAAACcFHioVWhRu5L6tqrlpxb5D2EBAAAAOMfS7Ye18/Bpp20P9m6mkJDg6lWQCAsAAACAg2EYevN753UVmtWsqGsuru2nFvkXYQEAAAD4/77bkaYdqaectj0QpL0KkhTm7wYAAABJW7+Qdi6U7Ln+bgngfX2el2pc5O9WuDnbq/Cb07YmNWJ0Xes6fmqR/xEWAADwt9+WSl+O8ncrAN/p/JC/W+BR8q40bfvTpVehVzOFBmmvgsQwJAAA/G/P9/5uARD0DMPQm985z4DUuHqM+l9W108tMgfCAgAA/pZzquRjAHjV8t+OaEvKSadt9/dsGtS9ChLDkAAA8L8854WflNBdSujmn7YAvhAX7+8WODlbq+Dcq9CwWrQGXB7cvQoSYQEAAP/Ly3T+uWkfqfOD/mkLEIRW7D6qTX+ccNp2f8+mCgtlEA5/AwAA+JtrWIiI8U87gCDkqVahftUKGtSmnp9aZC6EBQAA/M0tLFT0TzuAILRqT7rWHzjutO3+Hk0VTq+CJMICAAD+51qzQM8C4DOutQr1KlfQDW3NVVPhT4QFAAD8jWFIgF+s3puuNfuOOW27r2cTRYTxK3Ih/iYAAPA3wgLgF2+59CrUiYvSP9rRq3AuZkMCAJhanr1AX205pP1HM0s+OBAZBXrsjPN7S1p3REe37/JTg4DgkJFr18970p223dujiSLDQv3UInMiLAAATO2puVv15cYUfzfDa6KVo8einLd9uC5NKYZ/2gMEq1qxkbq5fX1/N8N0GIYEADCt/AJDC3455O9meFW0ct22ZRlRHo4E4E33dG+iqHB6FVwRFgAAppWZZ1eevcDfzfCqaFuO27ZMERYAX2pcPUa3dmzg72aYEsOQAACmlZlrd9t27SW1LbWqar2cLOnA/34uUIj6XtpQstn81yggiNSrXEHD/9aQXoUiEBYAAKblKSy8fWsbS4UF/ZErTf/fjyGRFfX2kLb+aw8AnMNCT1sAgNVk5OY7/RwZFmKtoCCxIBsAU7PYExcAYCWuPQsVIy3YIc4aCwBMjLAAADAt17AQQ1gAAJ8iLAAATCszzzksREdYsADRLSxU9E87AMADwoKJ7d+/X2+//bZuvPFGNWvWTNHR0YqKilJ8fLwGDhyomTNnym53L/4DAKtwrVmw5jAkahYAmJcFn7rW8M9//lMvvviiDMN9Cc8///xTf/75p+bPn6/XX39dX3zxhRo0YG5gANbDMCQA8C96FkwqNTVVhmEoJiZGQ4cOVWJiolasWKH169frk08+UYcOHSRJ69atU58+fZSRkVHCFQEg8FDgDAD+RVgwqWrVqumVV15RamqqPvnkE40YMUKdO3dWu3btNHToUK1atUo333yzJOn333/X66+/7ucWA0D5y3DrWbBizYLrMCRqFgCYB2HBpF555RWNGzdOlSpV8rg/NDRUkydPVkREhCTpiy++8GXzAMAnGIYEAP5FWAhg1apV06WXXipJ2rNnj59bAwDlLzPPucA5JoKwAAC+RFgIcLm5uZLO9jQAgNUEZ88Cw5AAmAdhIYClpaVpx44dkqSWLVv6uTUAUP7cC5wt+MUIU6cCMDELfkUTPCZOnOhYZ6Gw2Pl8pKSkFLs/NTX1gtoFAOXFdZ2F4OhZICwAMA8LPnWDw5o1a/TGG29IkuLj43Xvvfee9zXq169fzq0CgPLFMCQA8C+GIQWgv/76S//4xz9kt9tls9n00UcfKTo62t/NAoByFxzrLDAMCYB5WfCp61s2m63M10hMTNSIESNKdezp06d13XXXOYYQTZgwQb169bqg+x48eLDY/ampqerYseMFXRsAyoP7OgsW+2fLMNx7FsL58geAeVjsqWttOTk5GjBggDZs2CBJeuyxxzRu3LgLvl58fHx5NQ0Ayp09v0C59gKnbTERFitwzs+TCpwDEcOQAJgJYaGMCmcjKos6deqUeIzdbtfNN9+s5ORkSdKdd96piRMnlvneAGBWrmssSBbsWXDtVZAYhgTAVCz21PW9Fi1aeP0eBQUFGjZsmBYsWCBJuuWWWzRlyhSv3xcA/Mm1XkEiLACAr1HgHABGjx6tmTNnSpL69++vTz/9VCEh/KcDYG0ew4LVhiERFgCYHL9xmtwjjzyiDz/8UJLUu3dvzZkzR2FhFvtmDQA8cC1ujgoPUVioxf7Zcg0LYRWkEIsFIgABzWJPXWt5/vnnNWnSJElSp06dNH/+fEVGRvq5VQDgG5kuC7IxbSoA+J4Fn7zW8Pbbb+uFF16QJNWrV0+vvvqq9u3bV+w5F110kcLDw33RPADwusw8i0+bKrF6MwDTs+CT1xq+/PJLx+s///xTXbp0KfGcffv2qVGjRl5sFQD4jmvNQnSEBf/JYvVmACbHMCQAgCm5r95swbH8DEMCYHIW/JrGGpYtW+bvJgCAX2W41CwwDAkAfI+eBQCAKbn2LBAWAMD3CAsAAFNynTq1oiVrFlyHIVGzAMBcCAsAAFOiZwEA/I+wAAAwpaw815oFKxY4ExYAmBthAQBgSq7DkKzZs8AwJADmRlgAAJgSw5AAwP8ICwAAU3IrcGYYEgD4HGEBAGBKmXkuPQuWnA2JFZwBmBthAQBgSpkui7JVtOQwJFZwBmBuhAUAgCkFR4Ezw5AAmBthAQBgOmfyC5RnL3DaFhxTpzIMCYC5EBYAAKaT5TIESbJgz0JBvmTPdt5GzwIAkyEsAABMJ8OluFmyYFhw7VWQpIho37cDAIpBWAAAmI7rGguSBWdD8hgWGIYEwFwICwAA03Etbq4QHqrQEJufWuMlZ7LctzEMCYDJEBYAAKYTHKs3u0ybGhImhUb4py0AUATCAgDAdNzXWAiGmZBiJJvFek8ABDzCAgDAdFx7FqKtVq8gMW0qgIBAWAAAmE6my2xIrN4MAP5BWAAAmI776s1BMgwJAEyGsAAAMJ3gKHBmGBIA8yMsAABMx73A2YphgWFIAMyPsAAAMB33YUhWDAsMQwJgfoQFAIDpZLkUOMdEULMAAP5AWAAAmE6GyzAka/YsuA5DomYBgPkQFgAAphOcBc70LAAwH8ICAMB0XMOCNQucCQsAzI+wAAAwneAscGYYEgDzISwAAEzHfRiSFQucmToVgPkRFgAAphMc6ywwDAmA+REWAACmkmcvUF5+gdO26AjCAgD4A2EBAGAqrmssSMHSs0DNAgDzISwAAEzFtbhZsmDNgmFQswAgIBAWAACm4lqvIEkxVhuGZM+RDOehVoQFAGZEWAAAmIprz0J0RKhCQmx+ao2XuA5BkqRwwgIA8yEsAABMJThWb85w30bPAgATIiwAAEwlOFZvznLfFh7t+3YAQAkICwAAU8nMc65ZiI6wWHGz5D4MKTxGCuGfZADmw5MJAGAqQTkMiSFIAEyKsAAAMBXXAmdrDkNiQTYAgYGwAAAwleDoWWBBNgCBgbAAADAV9wJnK9YsMAwJQGAgLASgxYsXy2azOf48//zz/m4SAJSbDJdF2Sy3IJvEMCQAAYOwEGAyMzN17733+rsZAOA1WXnBOAyJsADAnAgLAeaf//ynDhw4oJo1a/q7KQDgFa4FzjFBMQyJmgUA5kRYCCAbNmzQW2+9pcjISL344ov+bg4AeEVwFjjTswDAnAgLASI/P1933XWX8vPz9fTTT6tp06b+bhIAeEWmS80CU6cCgP8QFgLEpEmTtGnTJjVv3lxPPPGEv5sDAF7jNgwpKAqcGYYEwJwICwFg//79Gj9+vCTpvffeU2RkpJ9bBADekxkUBc5MnQogMBAWAsC9996rrKws3XbbberVq5e/mwMAXuW+zoIVwwLDkAAEBgs+ga1lxowZWrJkiSpXrqzXX3+9XK+dkpJS7P7U1NRyvR8AlCTPXqAz+YbTtmhLzoZEWAAQGAgLJnbs2DE9/PDDkqSXX3653KdLrV+/frleDwDKyrVXQQqWngVqFgCYE8OQTOyxxx5TWlqarrjiCt19993+bg4AeJ1rcbNEzQIA+JMFn8C+ZbPZynyNxMREjRgxwmnbsmXLlJiYqNDQUL3//vsKCSn/XHfw4MFi96empqpjx47lfl8AKIprcbMkRYczDAkA/IWwYEK5ubkaPXq0JOnBBx/U5Zdf7pX7xMfHe+W6AHCh3BZkiwhVSEjZv5QxlfwzUn6u8zaGIQEwKcJCGe3YsaPM16hTp47Tz3PnztVvv/2m8PBwtWrVSjNnznQ7Z/v27Y7X27ZtcxxzxRVXKCEhocxtAgB/yHBZkM2aQ5Ay3bfRswDApCz4FPatFi1alPs1c3PPfuN05swZ3XXXXSUe/+WXX+rLL7+UdHZIE2EBQKAKymlTJSki2vftAIBSoMAZAGAarmEhKKZNlaRwehYAmJPfvrI5deqUTp8+rfz8/BKPbdCggQ9aZB4jRoxwK3h2tWzZMvXs2VOSNH78eD3//PPebxgAeJl7zYIVexZcZkIKjZDCIvzTFgAogU+fwt9++60mT56sFStW6NixY6U6x2azyW53nx0DAGA9mXnOXyBZchjSmSznn6lXAGBiPnsKP/jgg3r33XclSYZhlHA0ACAYua6zEBQFzsyEBMDEfPIUnjFjht555x1JUlRUlAYOHKh27dqpatWqXlk/AAAQmNyGIVkyLLAgG4DA4ZOn8JQpUyRJ9evX1w8//KAmTZr44rYAgADj2rNQMRgKnAkLAEzMJ2Hhl19+kc1m0/jx4wkK5aRHjx4M5wJgOVnBuM4CYQGAiflkDNCZM2ckSW3atPHF7QAAASozLwhnQ6JmAYCJ+SQsNGrUSJKUkZFR/IEAgKAWnAXO9CwAMC+fhIUbbrhBkvT999/74nYAgADlXuBMzQIA+JNPwsKjjz6qBg0a6I033tDOnTt9cUsAQADKzA2CdRaYOhVAAPFJWIiLi9M333yjWrVqqVOnTpo8ebKOHz/ui1sDAAJIcAxDYupUAIHDJ0/hxo0bS5KysrJ04sQJPfDAA3rwwQdVvXp1RUdHF3uuzWbTnj17fNFMAIAfGYbhNgwpOHoWCAsAzMsnT+H9+/c7/WwYhgzDUFpaWonn2mw2L7UKAGAmefkFshc4TwkdHUHNAgD4k0/Cwu233+6L2wAAAphrvYJk1Z4Fpk4FEDh88hROTEz0xW0AAAHMdQiSZNWaBXoWAAQOnxQ4AwBQEtfiZpuNYUgA4G+EBQCAKbitsRARZs26NaZOBRBA/NK/m52drQ0bNujw4cPKysrSwIEDFRsb64+mAABMwn3aVAv2KhQU0LMAIKD4NCwcPHhQTz/9tObMmaMzZ844trdv316tWrVy/Dxt2jRNmTJFcXFxWrp0qTW/WQIAOHEtcLZkvYI9W5LzjE+EBQBm5rNhSGvWrFGbNm00Y8YM5eXlOaZP9aR///765Zdf9MMPP2jp0qW+aiIAwI8y89yHIVmOa6+CxDAkAKbmk7Bw4sQJDRgwQMeOHVPt2rU1efJkbd26tcjja9asqWuvvVaStGjRIl80EQDgZ241C1YchuQ6baokhRe/OCkA+JNPvrZ56623lJaWpurVq2vVqlVq0KBBief06dNH8+fP19q1a33QQgCAvwXl6s2ySeEV/NIUACgNn/QsLFiwQDabTY888kipgoIkXXzxxZKkPXv2eLNpAACTyAiGmgVPMyFRlwfAxHwSFnbv3i1J6tatW6nPqVKliiTp1KlTXmkTAMBc3IchBUNYoLgZgLn5JCzk5ORIksLDw0t9Tmbm2QdqhQp0zwJAMAjKYUiEBQAm55OwULNmTUnSvn37Sn3O5s2bJUl169b1RpMAACYTlLMhERYAmJxPwsIVV1whSVq8eHGpjjcMQ1OnTpXNZlPXrl292TQAgEm4r7MQBLMhMW0qAJPzSVi47bbbZBiGPvvsM0ePQXEeffRRbdmyRZJ0++23e7l1AAAzcF/BmZ4FAPA3n4SFAQMGqGfPnrLb7erdu7fee+89paWlOfbb7XYdOnRIc+bMUdeuXfXmm2/KZrPphhtuUKdOnXzRRACAn1HgDADm47Mn8ZdffqnevXtr06ZNGjNmjMaMGSPb/58urk2bNk7HGoahK6+8UklJSb5qHgDAz9wLnK04DMnD1KkAYGI+6VmQpMqVK2vVqlV66qmnFBsbK8MwPP6pUKGCxo0bp2XLlikmhm9cACBYuA1DsmSBs2vNAv/OATA3nz6JIyIi9OKLL+rpp5/W8uXLtX79eqWlpSk/P1/VqlVTmzZt1KdPH8XFxfmyWQAAPzMMQ5l5wbgoG2EBgLn55UkcExOjfv36qV+/fv64PQDAZHLtBcovMJy2ERYAwP98NgwJAICiuNYrSEydCgBm4JevbbKzs7VhwwYdPnxYWVlZGjhwoGJjY/3RFACACbiusSCxgjMAmIFPn8QHDx7U008/rTlz5ujMmTOO7e3bt1erVq0cP0+bNk1TpkxRXFycli5d6pg1CQBgTa7FzSE2qUK4FXsWCAsAAovPhiGtWbNGbdq00YwZM5SXl+eY/ciT/v3765dfftEPP/ygpUuX+qqJAAA/ycxznwnJkl8UMXUqgADjk7Bw4sQJDRgwQMeOHVPt2rU1efJkbd26tcjja9asqWuvvVaStGjRIl80EQDgR0GxerPE1KkAAo5PnsZvvfWW0tLSVL16da1atUoNGjQo8Zw+ffpo/vz5Wrt2rQ9aCADwJ/fVmy04BEliGBKAgOOTnoUFCxbIZrPpkUceKVVQkKSLL75YkrRnzx5vNg0AYAJZuUGwxoI9Tyo447yNYUgATM4nYWH37t2SpG7dupX6nCpVqkiSTp065ZU2AQDMIyhXb5boWQBgej4JCzk5OZKk8PDwUp+TmXm2q7ZChQpeaRMAwDzchyFZMSxkum8jLAAwOZ+EhZo1a0qS9u3bV+pzNm/eLEmqW7euN5oEADCRDJfZkCpasWaBsAAgAPkkLFxxxRWSpMWLF5fqeMMwNHXqVNlsNnXt2tWbTQMAmEBQ9iyERUkhFgxFACzFJ2Hhtttuk2EY+uyzzxw9BsV59NFHtWXLFknS7bff7uXWAQD8zXUFZ2uu3sy0qQACj0/CwoABA9SzZ0/Z7Xb17t1b7733ntLS0hz77Xa7Dh06pDlz5qhr16568803ZbPZdMMNN6hTp06+aCIAwI+ComfhTJbzz4QFAAHAZ0/jL7/8Ur1799amTZs0ZswYjRkzxrE6Z5s2bZyONQxDV155pZKSknzVPACAH7mu4BwdYcHhOazeDCAA+aRnQZIqV66sVatW6amnnlJsbKwMw/D4p0KFCho3bpyWLVummBi+dQGAYJDBMCQAMCWfPo0jIiL04osv6umnn9by5cu1fv16paWlKT8/X9WqVVObNm3Up08fxcXF+bJZASEzM1NJSUmaO3eudu7cqaNHj6py5cqqV6+eOnfurP79+6tv377+biYAXJCgGIbE6s0AApBfnsYxMTHq16+f+vXr54/bB5zk5GSNHDlSBw4ccNqelpamtLQ0bdq0ST/99BNhAUDAcg0L1uxZYBgSgMBjwaextXz33Xfq37+/cnJyVLlyZd1zzz3q0aOHatasqaysLO3YsUMLFy7UX3/95e+mAsAFc1vB2ZJhgWFIAAKPaZ7Gf/31lxYuXKijR48qISFBf//73xUdHe3vZvnVkSNHNHjwYOXk5Ojyyy/XkiVLVKtWLadjOnfurDvvvFN5eXl+aiUAlI1hGB6GIQVDgTNhAYD5+SQs7NixQ+PHj5fNZtOUKVNUuXJlp/1fffWVhgwZouzsbMe2+Ph4zZ8/X5dffrkvmmhKTz31lNLT0xUdHa158+a5BYVzRURE+LBlAFB+cu0FKjCct8VEmOa7rPJDWAAQgHwyG9K8efP0xRdf6NChQ25BIS0tTUOHDlVWVpbTrEgHDx5U//79lZGR4fmiFnf8+HHNmDFDkjR06FA1bNjQzy0CAO9wHYIkBcswJGoWAJifT8LC999/L5vNpr///e9u+yZPnqyMjAyFhYXp9ddf15YtW/Tqq68qJCREhw4d0tSpU33RRNNZuHCho6fl+uuvd2zPysrS7t27dfjwYRmGUdTpABAwXIcgScFS4EzPAgDz80lY+OOPPyS5L74mnV2szWazafjw4Ro7dqxat26txx57TKNGjZJhGPrqq6980UTTWb16teN169attW7dOvXt21eVKlVSs2bNVKdOHdWqVUtjxoyhuBlAQHPtWQixSVHhPlsGyHcICwACkE++uklLS5Mk1axZ02n70aNH9euvv8pms2nIkCFO+66//npNnTpV27dv90UTTefc952cnKw777xTdrvzP6hHjhzRu+++qy+//FJLlizRZZdddl73SElJKXZ/amrqeV0PAC5EpsuCbDGRYbLZbH5qjRcxdSqAAOSTsFA4nCYnJ8dp+4oVKySdLc7t0qWL0746depIkk6cOOH9BprQsWPHHK/vuece2Ww2/d///Z+GDx+uWrVqaffu3Zo4caKSkpJ0+PBhDRw4UFu2bFFsbGyp71G/fn1vNB0AzktQrLEgMXUqgIDkk37eqlWrSvrfcKRC33//vSSpffv2brP5FH6LXrFicH7zkpn5v2+gcnJyNG3aND3zzDOqX7++IiIi1KpVKyUmJuruu++WJO3fv1/vvfeev5oLABcsKNZYkBiGBCAg+SQsFA6PKZzdRzrb2zBnzhzZbDb16tXL7ZzC1YqLmy7UDGw2W5n/JCUluV03KirK8frSSy/VsGHDPN7/pZdeUmRkpCRp1qxZ59X2gwcPFvtn7dq153U9ALgQWXkuYSHCgmssSAxDAhCQfPL1zeDBg7V06VItWLBAgwcPVpcuXTRr1iylpaUpJCREt956q9s5a9askaSgnTK0UqVKjtd9+/Yt8rhq1aqpffv2WrlypbZs2aK8vLxSr7kQHx9f5nYCQFlleKhZsJyCfOlMlvM2ehYABACfPJGHDx+u6dOna8WKFZozZ47mzJnj2Ddy5Ei1aNHC7Zy5c+fKZrOpU6dOvmjiBduxY0eZr1FYn3Gu+vXrO2ZEKqm2oHB/QUGBjh07ptq1a5e5TQDgK+6rN1swLLgGBYmwACAg+OSJHBISosWLF2v8+PGaM2eODh8+rDp16uj222/XP//5T7fjFy5cqP3798tms6lfv36+aOIF8xR0ysPFF1/sCFX5+fnFHnvu/rAwC/4jC8DSgqLA2XUIksQwJAABwWdP5JiYGP3nP//Rf/7znxKP7dy5s/bt2ycpeIchdevWzfF67969xR67Z88eSWfrHAqLyQEgULgXOFuwZsFjWKBnAYD5mXLVmypVqqhhw4ZBGxSks2GhRo0akqQFCxYU2buwb98+bd68WdLZkBUSYsr/pABQpKAYhuQ6bWpImBRauvoyAPAnfrM0qdDQUD322GOSzs4M9e9//9vtGLvdrvvuu08FBQWSzq7HAACBJjPP+cuQihFWDAsuPQvhMZIVF54DYDmEBRN78MEH1bZtW0nSCy+8oFtvvVVLlizRxo0bNWfOHHXr1k1LliyRJPXr10833nijP5sLABfEtWch2pI9C6yxACAwWfCJbB1RUVFauHCh+vfvrw0bNmjmzJmaOXOm23H9+vXTzJkzZeNbKgAByL3AOQhqFggLAAIEPQsmV6dOHa1evVrvv/++unfvrho1aig8PFy1a9fW9ddfr7lz52rRokVO6zIAQCAJihWcCQsAApQFn8jWExYWptGjR2v06NH+bgoAlLvMYFiUjdWbAQQoehYAAH4VHOssuMyGRM8CgABBWAAA+I1hGMrMcxmGFAyzIREWAAQIwgIAwG9yzhSowHDeFhSLshEWAAQIwgIAwG9ci5slq9YsuA5DomYBQGDwSVhISEhQkyZNtHv37lKf88cff6hx48Zq0qSJF1sGAPAn13oFyao1C/QsAAhMPnkiHzhwQDabTXl5eaU+58yZM9q/fz9rBwCAhbn2LISG2BQZZsFOb8ICgABlwScyACBQuPYsxESEWvNLIqZOBRCgTBsWTp48KUmKjo72c0sAAN7iOhOSJYcgSUydCiBgmTYsfPrpp5Kkhg0b+rklAABvyQiGBdkkhiEBCFheeSr36tXL4/aRI0cqJqb4B2Rubq727t2rtLQ02Ww29e3b1xtNBACYQJbLMKTooAkLDEMCEBi88lRetmyZbDabDON/k2cbhqF169ad13UaN26sp556qrybBwAwCdcC54pWXGNBomcBQMDySljo1q2bU4Ha8uXLZbPZ1K5du2J7Fmw2m6KiolSnTh116tRJgwcPLrEnAgAQuDJdhyFZcfVmw6BmAUDA8lrPwrlCQs6WRiQlJalVq1beuCUAIAAFRYGzPVcynEMRYQFAoPDJU3n48OGy2WyqUqWKL24HAAgQrsOQLFng7DoESaJmAUDA8MlTOSkpyRe3AQAEGLd1FiwZFjLct9GzACBAmOqpvGfPHh09elSNGjVSrVq1/N0cAICXuYYFSxY4e+pZCGcNIQCBwSfrLKSlpWny5MmaPHmyY7G1c+3evVvt2rVT8+bN1alTJ9WrV0833nijjh8/7ovmAQD8xLXAOdqKBc6uYSE8Rgox7TJHAODEJ0+ruXPnasyYMXrzzTcVFxfntC83N1fXXnutNm/eLMMwZBiGCgoKNG/ePA0YMMAXzQMA+ElQFDi7zYRErwKAwOGTsLB06VLZbDYNGjTIbV9SUpL27NkjSbr++uv15ptvqn///jIMQytXrtSsWbN80UQAgB8EZYEz9QoAAohPwsKuXbskSVdeeaXbvhkzZkg6u+rzvHnz9MADD2j+/Pnq06ePDMPQzJkzfdFEAIAfuBc4B0HNAjMhAQggPgkLR44ckSTFx8c7bc/Oztbq1atls9l09913O+274447JEkbN270RRMBAH7gWrNgyWFIZ+hZABC4fBIWTpw4cfZmLgVdq1ev1pkzZ2Sz2dSnTx+nfQkJCZLOFkcDAKzHMAy3mgWGIQGAufgkLFSseLbL9fDhw07bC1d6btWqlduCbeHh4ZKksDAL/sMBAFD2mXwZhvM2S/YsEBYABDCfhIUWLVpIkpYsWeK0/csvv5TNZlP37t3dzikMFqy3AADW5FrcLEnREVasWXCdDYmaBQCBwydf4Vx33XVavXq1PvjgA7Vs2VJdu3ZVUlKStm/fLpvNphtuuMHtnMJahXr16vmiiQAAH3OtV5AYhgQAZuOTp/KYMWM0efJkpaamasyYMU77/va3v6lnz55u5yxYsEA2m00dOnTwRRMBAD7mOhNSWIhNkWEWXKyMsAAggPnkqRwXF6fvvvtObdu2dSy8ZhiGunbtqtmzZ7sdv2XLFq1bt06SdNVVV/miiQAAH/O0xoLNZvNTa7yIqVMBBDCf9fe2bNlS69ev1759+3T48GHVqVNHjRo1KvL4xMRESWfXXwAAWI9rz4Ili5slDzUL9CwACBw+fzInJCQ4pkUtymWXXabLLrvMRy0CAPiDe8+CBYubJYYhAQhoFhwcCgAIBFl5zgXO0RFW7VlgGBKAwOXzJ3NBQYGSk5O1atUqHT58WFlZWXrxxRdVp04dxzF5eXmy2+0KDQ1VZGSkr5sIAPABhiEBgPn59Mm8cOFCPfjggzpw4IDT9scee8wpLHz44Yd64IEHVLFiRR06dEgxMTxYAcBqGIYEAObns2FIU6dO1YABA7R//34ZhqFq1arJcF268/+78847FRcXp4yMDP33v//1VRMBAD7k2rNgyTUWJMICgIDmk7Dw+++/6/7775d0dnaj7du3Ky0trcjjIyIidOONN8owDC1dutQXTQQA+FiGy6JslhyGlG+X7DnO26hZABBAfBIWJk2aJLvdrosvvlhff/21WrRoUeI5Xbt2lSRt2rTJ280DAPhBUPQsnMl030bPAoAA4pOw8MMPP8hms2ns2LGKiIgo1TlNmzaVJB08eNCbTQMA+ElQFDi7DkGSCAsAAopPwkJKSookndfaCYVFzVlZWV5pEwDAvzLznMNCdIQFC5w9hYVwwgKAwOGTsGCz2SSd3y/+6enpkqS4uDivtAkA4F+ZLjULlhyG5DptamiEFFa6HnYAMAOfhIV69epJkvbu3Vvqc1asWCFJaty4sVfaBADwr6AchsQQJAABxidhoUePHjIMQx999FGpjj958qTef/992Ww29erVy8utAwD4g/s6C0EQFhiCBCDA+CQsjB49WjabTcuXL1dSUlKxx6anp2vgwIE6fPiwwsLCdM899/iiiQAAH3PvWbBizQKrNwMIbD4JC23atNFDDz0kwzA0atQo3XLLLZo9e7Zj/88//6wZM2bo/vvvV9OmTfXjjz/KZrPpn//8pxo2bOiLJgIAfKigwFBmXjDULDAMCUBg89mT+bXXXlNubq7ee+89ffHFF/riiy8chc+jR492HFe4qvPYsWP17LPP+qp5prZkyRIlJSVp7dq1Onz4sAoKClSjRg21bdtWQ4YM0U033aSQEJ8txg0AZZZ9Jt9tW0yEFcOCy8QehAUAAcZnv2HabDa9++67+uabb9SjRw/ZbDYZhuH0R5L+9re/adGiRXr99dd91TTTys3N1T/+8Q9de+21mjVrlvbt26fs7Gzl5uYqJSVFX331lQYPHqwePXroxIkT/m4uAJSa6xAkyao9C67DkFi9GUBg8fmT+aqrrtJVV12l06dPa9OmTUpLS1N+fr6qVaumyy+/XNWrV/d1k0zrwQcf1JdffilJqlmzpsaNG6e2bdsqPDxcW7du1SuvvKIDBw7op59+0uDBg7VkyRI/txgASse1uFmSYixZs8AwJACBzW9f41SqVEndunXz1+1N76+//tKHH34oSapSpYo2bNig+Ph4x/4uXbrotttu02WXXab9+/frm2++0fr169W+fXt/NRkASs11jYXwUJsiwwgLAGA2DHQ3qTVr1qigoECSNHLkSKegUCg2NlYPP/yw4+dVq1b5rH0AUBZBMW2q5CEsMAwJQGAxTVg4fvy4jhw54qhdCHZ5eXmO18UtTNekSROP5wCAmbnWLFiyuFli6lQAAc+rYcFut2vbtm3asGGDjhw54rY/JydHzz33nOLj41W9enXVrl1blSpV0j/+8Q/9+uuv3mya6V100UWO18WtfL1nzx6P5wCAmWXmBcHqzRLDkAAEPK+EBcMw9Nxzz6l69eq67LLL1LFjR9WuXVtdunTRunXrJJ39Fvzqq6/Wiy++qNTUVMeMSFlZWfrvf/+rjh076vvvv/dG8wJC69at1alTJ0lSUlKSDh065HbM6dOn9cYbb0g62/vQt29fXzYRAC6Ya81CtBWLmyXCAoCA55WvckaOHKlPPvlEkpyGFf3888+65pprtGbNGk2ePFk//fSTJKlq1apq1qyZ7Ha7tm/fruzsbGVnZ+u2227Trl27FBcX541mml5iYqKuueYa7du3T23btnXMhhQWFqZt27bp1Vdf1b59+1S9enV99tlnioiIOK/rp6SkFLs/NTW1LM0HgCK5r95s1Z4Fpk4FENjK/emcnJysjz/+WDabTZGRkbruuuuUkJCgAwcOaOHChTpx4oQmTZqkzz//XOHh4Xr33Xc1atQoxwJt2dnZGj9+vP7zn//oyJEjSkpK0kMPPVTezQwIzZs317p16/Tee+/plVde0aOPPuq0Pzw8XI899pgeeughjwXQJalfv355NRUAzotbgbNlaxboWQAQ2Mr96ZyYmCjp7LoAP/zwg1q2bOnYt3PnTvXq1UsffPCBCgoK9Pjjj+vOO+90Or9ChQp69dVXtXXrVn3zzTdatGhR0IYFSVqwYIE+++wzZWRkuO07c+aMZs+erRo1aujxxx93BC4AMDu3AmfL9iwQFgAEtnKvWVizZo1sNpsefvhhp6AgSS1atNDDDz+s/PyzY1WHDRtW5HVuv/12STJ9obPNZivzn6SkJI/XfvTRRzVy5Ejt3LlTAwcO1MqVK5WRkaHs7Gxt3LhRI0eO1B9//KEnnnhC//jHPxx/r6V18ODBYv+sXbu2HP6GAMCde4FzsNQsMAwJQGAp969yCgtx//a3v3ncf+72pk2bFnmdZs2aSZKOHTtWjq0LHIsWLdLrr78uSRoxYoSjx6ZQmzZtNH36dMXHx+vf//635s6dq8mTJ+uBBx4o9T0uZOgSAJSHDJcCZ0v2LBgGU6cCCHjl/nTOzMyUzWZT1apVPe6vXLmy43VkZGSR14mKipJk/rUDduzYUeZr1KlTx21b4erNNptN//d//1fkuU8//bQmTZqkjIwMTZ8+/bzCAgD4S1AMQzqTLcll7SDCAoAA47Wnc1Hj5602rr5FixZeuW5hCKlZs6bq1atX5HFRUVG6+OKLtWbNGu3cudMrbQGA8ua+KJsFhyG5DkGSGIYEIOCYZgVnOAsLO5vj7HZ7CUeeLXQ+9xwAMDvXmgVL9iy4DkGS6FkAEHAICyaVkJAgSUpPTy92qNOxY8e0bds2p3MAwOxcF2Wz5DoLbj0LNim8gl+aAgAXymtP58mTJ6tmzZpu29PS0hyv//WvfxV5/rnHBaP+/ftr4cKFkqSxY8dqwYIFbouuFRQU6MEHH3TUdfz973/3eTsB4EK4rbMQDGEhoqJksaG4AKzPa0/n9957r8h9hXULL7zwgrduH/BGjBihN954Qzt27NDSpUvVvn17PfDAA7rssssUGhqq7du367333tOqVaskSbVq1dIjjzzi51YDQOkERYEzMyEBsACvPJ0Nwyj5IBQrIiJCixcv1oABA7RlyxZt3bpVd999t8djExISNHfuXFWvXt3HrQSA81dQYCgrLwiHIUVE+6cdAFAG5f50Tk5OLu9LBq2GDRtq3bp1mjlzpr744gtt3LhRR44ckWEYqlq1qi699FINHDhQw4cPV0wM31gBCAxZZ9wXkIyx4qJsrN4MwALKPSx07969vC8Z1MLDwzVs2LBiV7sGgEDiOgRJkmIirNiz4DoMiWlTAQQeZkMCAPiUa3GzZNGahTNZzj/TswAgABEWAAA+5dqzEBEaoogwC/5zxDAkABZgwaczAMDM3KdNtWC9guR56lQACDCEBQCAT7kuyGbJIUgSU6cCsATCAgDAp1yHIVly2lSJYUgALIGwAADwqcw857AQHREsw5AICwACD2EBAOBTQbF6s8TUqQAsgbAAAPCpjNwgWL1ZomcBgCUQFgAAPhU8PQuEBQCBj7AAAPCp4C1wZhgSgMBDWAAA+FTwrLPA1KkAAh9hAQDgUwxDAoDAQVgAAPhUZp7LomwRFgwL9jwpP895G8OQAAQgwgIAwKeComfhTKb7NnoWAAQgwgIAwKfcC5wtWLPgOgRJIiwACEiEBQCAT7mus2DJngVPYSGcsAAg8BAWAAA+FRTDkFxnQgqLkkIt+D4BWB5hAQDgM/kFhrLPBMEKzsyEBMAiCAsAAJ/JyrO7bbNmzwJhAYA1EBYAAD6T6VKvIEkxEUFQ4Ey9AoAARVgAAPiM6+rNklV7Fli9GYA1EBYAAD7jWtwcERai8FAL/lPEMCQAFmHBJzQAwKzc11iwYK+C5CEssHozgMBEWAAA+IzrMKQYKy7IJtGzAMAyCAsAAJ/JdJkNKSYiWHoWCAsAAhNhAQDgM66zIVmyuFkiLACwDMICAMBngmL1ZsnDbEjULAAITIQFAIDPuBc4U7MAAGZGWAAA+EyG6zAkahYAwNQICwAAnwmeYUhMnQrAGggLAACfycgLlnUWWMEZgDUQFgAAPhO8PQuEBQCBibAAAPCZLLepU4OlwJlhSAACE2EBAOAzbis4W7HAuaBAOkPPAgBrICwAAHzGbQVnKw5DOpPlvo2wACBAERYAAD7jvs6CBcOC6xAkiWFIAAIWYQEA4DNuw5CsWLPgOhOSRM8CgIBFWAAA+IQ9v0A5ZwqctgVFz4ItVAqL9E9bAKCMCAsAAJ/IOpPvts2SNQueZkKy2fzTFgAoI8ICAMAnXOsVJIvOhsQaCwAshLAAAPAJj2EhGGoWCAsAAhhhAQDgExkuC7JFhoUoLNSC/wzRswDAQiz4lPa/jIwM/fjjj/rPf/6jm2++WQkJCbLZbLLZbGrUqNF5X2/btm0aPXq0mjRpogoVKqhGjRrq2rWr3n//fdnt7t/UAYAZBcW0qRJhAYClWPRJ7V/9+/fXsmXLyuVaU6dO1ZgxY5SXl+fYlpOToxUrVmjFihVKTEzUokWLVL169XK5HwB4i/u0qRb9J4hhSAAshJ4FLzAMw/G6atWq6tu3rypWPP8Feb7++mvdc889ysvLU61atfTWW29pzZo1Wrx4sW644QZJ0tq1azVo0CDl57vPMgIAZuLas2DdsEDPAgDrsOiT2r+GDBmi0aNHq0OHDmratKkkqVGjRsrI8LBQTxHOnDmjBx54QAUFBYqNjdXKlSvVpEkTx/5rrrlG999/vyZPnqwVK1bok08+0YgRI8r7rQBAucnMc/5SIybCgsXNknQmy/lnwgKAAEbPghfcfffduvXWWx1B4UL897//1d69eyVJTz31lFNQKDRx4kRVqVLF8RoAzCx4ehZchyGdf88yAJgFYcGk5s2b53hdVI9BdHS0br75ZknS9u3b9dtvv/mgZQBwYShwBoDAQ1gwqRUrVkiSLrroItWuXbvI47p37+54vXLlSq+3CwAulHuBs0WHIREWAFgIYcGEMjIydPDgQUlSixYtij323P07duzwarsAoCyCZxiSa1hgGBKAwGXRJ3VgS0lJcbyOj48v9tj69es7XhcGjAu5jyepqanndT0AKE6my6Js1h2GxNSpAKzDok/qwHb69GnH65KmXI2J+d8/Qucz25LkHDQAwNuCZ50FhiEBsA6GIZlQTk6O43VERESxx0ZGRjpeZ2dne61NAFBWWXkuYcGqU6cyDAmAhVj0a52S2Wy2Ml8jMTHRK2sbREVFOV6fu3KzJ7m5uY7XFSpUOK/7lDRsKTU1VR07djyvawJAUTJchiFZt2eBYUgArMOiT+rAVqlSJcfrkoYWZWb+7xus810luqR6CAAoT0FR4GwYDEMCYCkWfFKXTnnMHFSnTp1yaIm7evXqOV6XVIR8bu8ANQgAzCwo1lnIz5MKnN8nw5AABDILPqlLp6QpSf2pUqVKql+/vg4ePKidO3cWe+y5+1u2bOntpgHABQuKAmfXXgWJngUAAY0CZ5Pq0qWLJGnXrl06fPhwkcctX77c8bpz585ebxcAXAh7foFy7QVO2ypacVE213oFibAAIKARFkxq4MCBjtdJSUkej8nKytLs2bMlSa1atVLz5s190DIAOH+uayxIQdSzEB7t+3YAQDkhLJjUoEGD1LhxY0nSyy+/rD179rgd8/jjj+v48eOO1wBgVpku06ZKUnREEISF8GgpxII9KACChgWf1P63e/durVixwmlb4axGGRkZbj0F11xzjWrXru20LTw8XG+//bb69++vU6dOqXPnznr22WfVsWNHHT9+XFOnTtWXX34p6eyQpWHDhnnvDQFAGbkWN0sWXWeBaVMBWAxhwQtWrFihkSNHetyXnp7uti85OdktLEhSv3799P7772vMmDH666+/9MADD7gd07FjR/33v/9VaKgF/9EFYBmuxc1R4SEKC7Vg5zbTpgKwGAs+qa3lrrvu0oYNG3TXXXepcePGioqKUrVq1dSlSxe99957WrlypapXr+7vZgJAsVxrFiw5barE6s0ALMeiT2v/GjFiRLmu7HzJJZfogw8+KLfrAYCvBcW0qRLDkBAwsrOzderUKWVmZio/330CAphHaGioYmJiFBsbqwoVKvj8/hZ9WgMAzMRt9WYrFjdLngucAZM5efKkDh065O9moJTsdrtyc3N17Ngx1a1bV3FxcT69v0Wf1gAAM8nKc+1ZsGidFTULMLns7Gy3oBAWxq+DZma3/+/5eejQIUVGRioqKspn9+fTAQDwugyXmoXgGYZEzQLM5dSpU47XsbGxql27NpOkmFx+fr4OHz7s+G938uRJn4YFCpwBAF7nNgzJsmEhy/lnehZgMpmZ/+v9IigEhtDQUKdZM8/9b+gLhAUAgNe5FjhXDJaaBcICTKawmDksLIygEEBCQ0Mdw8V8XZBOWAAAeF3w9CwwDAmAtRAWAABel+lS4FyRAmcACAiEBQCA1wVPgTNhAYC1EBYAAF6X5TIMKTpowgLDkAAENsICAMDr3AqcLTsMiRWcgUByxx13yGazqVq1asrNzS322M2bN+uee+5Rq1atFBsbq4iICNWuXVtXXXWVXnvtNR05csTtHJvN5vQnLCxMderU0cCBA/Xjjz96622VK4t+tQMAMBPXmoWgWcGZsACY1unTpzV79mzZbDYdO3ZM8+bN0y233OJ2XEFBgcaNG6fXXntNoaGh6tatm/r27auYmBilpaVp1apVeuyxxzR+/Hjt2rVL9erVczq/WrVqGjNmjCQpJydHmzdv1vz58/XVV19p1qxZuummm3zyfi+URZ/WAAAzyXSpWajIMCQAfjZr1ixlZmbqkUce0RtvvKFp06Z5DAvPPPOMXnvtNbVt21azZs1S06ZN3Y7ZuHGjnnjiCWVnZ7vtq169up5//nmnbR9++KHuuusujRs3zvRhgWFIAACvcx2GZMkC54J8ye7yiwI9C4BpTZs2TWFhYRo3bpx69uyp77//XgcOHHA65rffftPEiRNVo0YNLVmyxGNQkKS2bdvq22+/VaNGjUp17zvuuEMxMTHav3+/x+FLZmLBpzUAwEzO5Bcoz17gtM2SYcG1V0EiLCBgFBQYOp6V5+9mlFqV6AiFhNgu+Pzt27dr9erV6tevn2rVqqXhw4fr+++/V2JiolMvwEcffaT8/HyNHj1aNWrUKPG6hQunnQ+b7cLfhy9Y8GkNADAT1wXZJIsOQyIsIIAdz8pTu//7zt/NKLUNz/ZRtYqRF3z+tGnTJEnDhg2TJN1www267777lJiYqOeee04hIWcH36xatUqS1LNnzzK22NlHH32kzMxMJSQkqHr16uV67fJmwac1AMBMMvPy3bZFW3E2JI9hgZoFwGzOnDmjTz75RLGxsRo4cKAkqWLFiho0aJA+/fRTfffdd+rbt68k6fDhw5KkunXrul1n2bJlWrZsmdO2Hj16qEePHk7bjh496uityMnJ0ZYtW7RkyRKFhIRo4sSJ5frevIGwAADwKk89C5acDcl12tSQcCkswj9tAVCk+fPn68iRIxo1apSioqIc24cPH65PP/1U06ZNc4SF4ixbtkwvvPCC23bXsJCenu44LjQ0VNWrV9eAAQP06KOPqmvXrmV7Mz5AgTMAwKtci5srhIcqtAxjjU2LaVOBgFA4BGn48OFO23v37q169epp/vz5OnbsmCSpVq1akqRDhw65Xef555+XYRgyDEOff/55kfe76KKLHMfZ7XYdPnxY8+bNC4igINGzAADwMteeBUsWN0tMm4qAViU6Qhue7ePvZpRalegL67U7ePCgli5dKknq3r17kcd9+umnevDBB9WpUyctW7ZMycnJ6tWr1wXdM9BZ9IkNADAL17DA6s2A+YSE2MpUMBwokpKSVFBQoC5duuiiiy5y22+32/XRRx9p2rRpevDBB3X77bdrwoQJ+uCDD/TQQw+ZvhjZGwgLAACvynBZkC14ehYIC4CZGIahxMRE2Ww2ffTRR2rcuLHH43777TetWrVK69evV/v27TVu3DhNmDBB1157rT7//HOPay2cOHHCy633H4s+sQEAZpGVF6zDkAgLgJn88MMP2rdvn7p3715kUJCkkSNHatWqVZo2bZrat2+vF198UXl5eXr99dfVokULdevWTZdddpmio6OVlpamX375RWvXrlXFihV1+eWX++4N+QgFzgAAr3JbvTmCYUgAfK+wsHnEiBHFHnfLLbeoQoUK+vzzz5Wdna2QkBC99tpr2rhxo0aNGqXU1FR9+OGHmjhxohYsWKCKFStq4sSJ2rNnj2MqViux6Nc7AACzCN4CZ8ICYCYzZszQjBkzSjwuNjZWWVlZbtvbtGmjKVOmnNc9DcM4r+PNiJ4FAIBXZbrULFhy9WaJsADAkggLAACvchuGZNWwcIapUwFYD2EBAOBVDEMCgMBFWAAAeJVrz4J111kgLACwHsICAMCrsvKcaxaiI4KlZ4FhSAACH2EBAOBV7is4WzUsMHUqAOshLAAAvCpoCpwZhgTAgggLAACvci9wpmYBAAIFYQEA4FXBu84CNQsAAh9hAQDgNXn2AuXlFzhts+QwJMOgZgGAJREWAABe4zoESbJoz4I9RzKcQxFhAYAVEBYAAF6TmeceFqIjLFiz4DoESWIYEgBLICwAALzGtV5BkmKsuM6C6xAkiZ4FAJZAWAAAeI3rtKnREaEKCbH5qTVe5NazYJPCKvilKQBQnggLAACvcZ821YK9CpLnaVND+CcWMKP9+/fLZrPJZrOpdu3astvdh0tK0o4dOxzHNWrUyLE9KSnJsb2oPyNGjHC6VmZmpl566SW1bdtWFStWVGRkpOLj49W1a1c99dRT2rNnjxffcdlY9KmNYJV2Kkcz1x3Uscw8fzcFgKQ/jmU5/WzJ4maJmZCAABQWFqa//vpLX3/9ta6//nq3/dOmTVNIMaG/d+/e6tKli8d9l19+ueP16dOn1aVLF/3yyy9q2rSphg4dqmrVquno0aNau3atJkyYoCZNmqhJkyZlfk/eYNGnNoLRmr3puu+zjUonKACmxYJsAMyiU6dO2rJli6ZPn+4WFux2uz799FP16dNHy5cv93h+nz599OSTT5Z4nzfeeEO//PKL7rzzTn3wwQey2ZyHYu7bt0+5ubkX/ka8jD5SBDzDMPTxqv267cM1BAXA5OIqhPu7Cd5BWAACToUKFTR48GAtWrRIaWlpTvsWLlyov/76S3fccUeZ77Nq1SpJ0v333+8WFCQpISFBLVq0KPN9vIWeBS/IyMjQxo0btXbtWq1du1br1q3T/v37JUkNGzZ0vC5OQUGBVqxYoSVLlujnn3/Wzp07dezYMUVFRalBgwbq1q2b7rnnHl166aXefTMml2vP13PzftWs9Qf93RQApXD1xbX93QTvcBuGxLSpCDAFBVL2MX+3ovQqVC2XuqA77rhDU6ZM0SeffKJHH33UsX369OmqWrWqBg4cWOZ7VKtWTZL022+/OQ1PChSEBS/o37+/li1bVqZrNGrUSAcPuv8CfObMGf3666/69ddfNWXKFD322GOaMGGCx6RqdWmncnTPpxu08Y8Tbvs6NqqqJjX5Zg8wixCbTR0Tqur6y+r6uyneQc8CAl32MWmiOcfMe/T4Himmepkv07FjR11yySVKTEx0hIXDhw9r8eLFuvfeexUZGVnkud99951ycnI87hs8eLCjt+Cmm27Sp59+qjvvvFNr165V37591a5dO0eIMDvCghcYhuF4XbVqVbVv314///yzMjI8zMNdhEOHDkmSmjZtqhtvvFGdO3dW3bp1lZ2dreTkZE2aNEnHjx/Xq6++qtDQUL300kvl/j7MbNMfxzX6kw1KO+0+xm9Mz6Z6+KrmCrXi9IwAzMk1LIRH+6cdAM7bHXfcoUceeURr1qzRFVdcoY8++kh2u73EIUjff/+9vv/+e4/7Lr/8ckdYuP766/Xaa69p/Pjxeu211/Taa69Jkpo0aaJrrrlGDz30kJo1a1a+b6ocUbPgBUOGDNGMGTP0+++/Kz09Xd988815p8eOHTtqyZIl+u233zRhwgT1799f7dq1U5cuXfTPf/5T69atU40aNSRJEydO1N69e73xVkxp9vqDumXKaregUCE8VJNva6vHrr6IoADAt9x6FhiGBASKoUOHKjw8XNOnT5ckJSYmqk2bNiUOGXr55ZdlGIbHP67Dlx555BEdOnRIs2fP1tixY9WlSxf98ccfevfdd3XppZfqq6++8tK7KzvCghfcfffduvXWW9W0adMLvsbPP/+sq6++usjhRU2aNNFzzz0n6WzF/rx58y74XoHiTH6Bnv/qV4374hfl5Rc47atftYLm3tdJ/VrX8VPrAAQ1pk4FAlaNGjXUv39/zZw5U99995127dpVLoXNripVqqSbbrpJkyZN0k8//aQjR47ovvvuU05OjkaNGqW8PHNO0sIwpADWs2dPx2szL+ZRHtIzcnX/jI1avde9+KpTk2p6d0hbVYmJ8EPLAEDULCDwVah6tg4gUFSoWq6XGzVqlObOnasRI0YoKipKt912W7le35O4uDi98847WrRokQ4cOKCtW7eqXbt2Xr/v+SIsBLBz5+QNDbXo3OWSfj10Und/vEF/nsh223dH5wQ93a+FwkLpJAPgR3nOi88xDAkBJySkXAqGA9XVV1+tevXq6c8//9TgwYNVpUoVn9zXZrMpJsbcXy4QFgLYuYuEtGzZ0o8t8Z4FWw7p8S+2KOeM87CjiLAQvTyotW5sF++nlgHAORiGBAS00NBQzZs3TykpKeU+vemUKVPUtm1bdejQwW3fvHnztGPHDlWuXFmXXHJJud63vBAWAlRWVpbeeOMNSVJkZKQGDBhw3tdISUkpdn9qauqFNK1c5BcYmvjNLr2/3L1LtHZslKYMa6fL6lf2fcMAwBOGIQEBr3379mrfvn2pjy9u6tTatWvrnnvukSQtXrxY99xzj5o2beqY3TIzM1ObNm3STz/9pJCQEE2ePLnYaVr9ibAQoJ544gn98ccfks6uCFi37vnPXV6/fv3ybla5OJl9Rg/N3KRlu4647WvfsIomD22rmpWi/NAyACgCYQEIOsVNnXrZZZc5wsIrr7yizp0769tvv9WPP/7o+DK2Xr16uv322/XAAw+YslahkM04d1EAeE2jRo104MCBUq/gXJzPPvtMQ4cOlXR2+NGGDRtUoUKF877O+SzkdvDgQcXHe3nIT+ovOnRglyYv26O0U+5JvWuzGhrcsYHCmRYVgNksHCtlnvMFx62zpIuu8VtzgKL8/vvvstvtCgsLM/Xc/nBXmv92KSkpji+Dy+t3t6DtWSiPFY8TExM1YsSIsjfmPCxbtkyjRo2SdHbBty+//PKCgoIkjytEnys1NVUdO3a8oGtfiIPfvqP6e2fp/yTJ08RGB/7/HwAwO3oWAFhE0IaFQLR+/Xpdf/31ys3NVcWKFfX111+XqbDZ6z0F52Frykn98ttR3cYnEoAVEBYAWETQ/mq2Y8eOMl+jTh3fLQD266+/6pprrtHp06cVGRmpefPm6YorrvDZ/b3tknqxyqpZUXJfRgEAAkt4jFTTmjPUAQg+QRsWWrRo4e8mlNqePXt01VVXKT09XWFhYZo1a5Z69+7t72aVK5vNpnatmip1VV3lnMlXxcgwVasYKcoTAASUSnWk7k9I4Rc2PBQAzCZow0KgSElJUZ8+fZSamqqQkBB99NFHFzRNaiAI6/OcQjuO04rtf2noFQ3Kpa4EAAAAF46wYGJpaWnq06ePY/ak999/X0OGDPFvo7ysZmyUhl3Z0N/NAAAAgKQQfzcAnp04cUJXX321du3aJUmaNGmS7rrrLj+3CgAAAMGEngUv2L17t1asWOG0LSMjw/G/SUlJTvuuueYa1a5d2/Fzbm6urrvuOm3evFmSdNttt6lPnz7atm1bkfeMiYlRQkJC+bwBAAAAQIQFr1ixYoVGjhzpcV96errbvuTkZKewkJqaqp9//tnx82effabPPvus2Ht2795dy5Ytu/BGAwAAywsNDZXdbpfdbld+fr5CQ0P93SSUQn5+vux2uyT5/L8Zw5AAAACCREzM/9YAOXz4sPLz8/3YGpRGfn6+Dh8+7Pj53P+GvkDPgheMGDGiTCs7N2rUSIZhlF+DAAAAJMXGxurYsbOLGp06dUqnTp1SWBi/DppZYY9Cobi4OJ/en08HAABAkKhQoYLq1q2rQ4cOOba5/jIK86pbt66ioqJ8ek/CAgAAQBCJi4tTZGSkTp48qczMTIYimVxoaKhiYmIUFxfn86AgERYAAACCTlRUlF9+8UTgocAZAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARBc4o0rlTqaWmpvqxJQAAACjJub+vldeUuIQFFOnIkSOO1x07dvRjSwAAAHA+jhw5okaNGpX5OgxDAgAAAOCRzTAMw9+NgDnl5ORo69atkqQaNWqwHHwRUlNTHT0va9euVZ06dfzcIgQzPo8wEz6PMJNg+Dza7XbHyJDWrVuXy1oa/PaHIkVFRalDhw7+bkZAqVOnjuLj4/3dDEASn0eYC59HmImVP4/lMfToXAxDAgAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHrEoGwAAAACP6FkAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QF4BxpaWlauHChnnvuOV177bWqXr26bDabbDabRowYcd7XW7x4sQYNGqT4+HhFRkYqPj5egwYN0uLFi8u/8bCc9evX61//+pf69u3r+AxVrFhRzZs318iRI7VixYrzuh6fR1yoU6dOaebMmXr00UfVvXt3NW3aVHFxcYqIiFDNmjXVo0cPvfrqq0pPTy/V9X7++WcNHTpUDRs2VFRUlGrXrq2rr75an3/+uZffCYLBE0884fi322azadmyZSWew/OxGAYAB0lF/rn99ttLfZ38/Hxj1KhRxV7vzjvvNPLz8733ZhDQunbtWuznp/DP8OHDjdzc3GKvxecRZfXtt9+W6vNYvXp1Y8mSJcVea/z48UZISEiR17juuuuM7OxsH70zWM2mTZuMsLAwp89UcnJykcfzfCwZPQtAERo0aKC+ffte0LnPPPOMpk2bJklq06aNPv/8c61du1aff/652rRpI0n68MMP9eyzz5Zbe2Ethw4dkiTVrVtXDz30kL744gutXbtWq1at0uuvv6569epJkj7++OMSe734PKI81K9fX8OHD9ebb76puXPnatWqVVq5cqVmzZqlm266SaGhoTp69Kiuv/56bdmyxeM1pkyZohdeeEEFBQVq0qSJpk2bprVr12revHnq2bOnJGnRokW64447fPnWYBEFBQW6++67ZbfbVbNmzVKdw/OxFPydVgAzee6554wFCxYYhw8fNgzDMPbt23fePQu7du1yfKvRvn17Iysry2l/Zmam0b59e0OSERYWZvz+++/l/TZgAdddd50xa9Ysw263e9x/5MgRo3nz5o7P5/Llyz0ex+cR5aGoz+G5/vvf/zo+j4MGDXLbn56ebsTFxRmSjAYNGhhHjhxxu0f//v1L9W0w4MmkSZMMSUaLFi2Mp556qsTPEs/H0iEsAMW4kLBw7733Os5ZtWqVx2NWrVrlOOa+++4rxxYjmCxYsMDxOXrggQc8HsPnEb500UUXOYYjuXrllVccn7PPP//c4/kHDx40QkNDDUlGv379vN1cWMiBAweMihUrGpKMZcuWGePHjy8xLPB8LB2GIQHlyDAMzZ8/X5LUokULXXnllR6Pu/LKK3XRRRdJkubPny/DMHzWRlhH4bANSdqzZ4/bfj6P8LVKlSpJknJyctz2zZs3T5IUGxurG264weP58fHx6tOnjyTp+++/1+nTp73TUFjO/fffr4yMDN1+++3q3r17icfzfCw9wgJQjvbt2+cYa17Sw6pw/59//qn9+/d7u2mwoNzcXMfr0NBQt/18HuFLu3bt0ubNmyWd/eXrXHl5eVq7dq0k6W9/+5siIiKKvE7hZzE3N1fr16/3TmNhKbNnz9bChQtVtWpV/ec//ynVOTwfS4+wAJSj7du3O167/mPp6tz9O3bs8FqbYF3Lly93vG7ZsqXbfj6P8LasrCz9/vvvev3119W9e3fZ7XZJ0tixY52O++2335Sfny+JzyLK14kTJ/TQQw9Jkl555RVVr169VOfxfCy9MH83ALCSlJQUx+v4+Phij61fv77j9cGDB73WJlhTQUGBJkyY4Pj55ptvdjuGzyO8ISkpSSNHjixy/5NPPqkhQ4Y4beOzCG8ZN26cDh8+rM6dO2vUqFGlPo/PZOkRFoBydO742ooVKxZ7bExMjON1RkaG19oEa5o0aZJjWMcNN9ygdu3auR3D5xG+dPnll+uDDz5Qhw4d3PbxWYQ3/PTTT/rwww8VFham999/XzabrdTn8pksPYYhAeXo3KK+4sbkSlJkZKTjdXZ2ttfaBOtZvny5nnzySUlSzZo19d5773k8js8jvGHgwIHaunWrtm7d6piPftCgQdq8ebNuvfVWLVy40O0cPosob3l5ebr77rtlGIYefvhhXXLJJed1Pp/J0iMsAOUoKirK8TovL6/YY88tTq1QoYLX2gRr+fXXXzVo0CDZ7XZFRUVpzpw5RS4+xOcR3lC5cmVdcskluuSSS9ShQwcNHjxYc+fO1ccff6y9e/dqwIABSkpKcjqHzyLK20svvaSdO3eqQYMGGj9+/Hmfz2ey9AgLQDkqnDZQKrmrMjMz0/G6pC5QQDo7e0ffvn11/PhxhYaGaubMmerWrVuRx/N5hC8NGzZMN910kwoKCjRmzBgdO3bMsY/PIsrTzp079fLLL0uS3n77badhQqXFZ7L0qFkAytG5RVLnFk95cm6R1LnFU4Anhw4dUp8+fXTo0CHZbDZNnz5dAwYMKPYcPo/wtQEDBmj27NnKzMzUkiVLHIXOfBZRniZNmqS8vDw1btxYWVlZmjlzptsx27Ztc7z+4YcfdPjwYUlS//79FRMTw2fyPBAWgHLUqlUrx+udO3cWe+y5+z1NewkUOnr0qK666irt3btX0tlv0oYPH17ieXwe4Ws1atRwvD5w4IDjdfPmzRUaGqr8/Hw+iyizwmFBe/fu1a233lri8f/+978dr/ft26eYmBiej+eBYUhAOUpISFDdunUlOc+B78mPP/4oSapXr54aNWrk7aYhQJ08eVJXX321Y07wCRMm6P777y/VuXwe4Wt//vmn4/W5wzUiIiLUsWNHSdKqVauKHSNe+FmNjIxU+/btvdRSBDuej6VHWADKkc1mcwwN2blzp1avXu3xuNWrVzu+qRgwYMB5TfeG4JGVlaXrrrtOGzdulCQ988wzeuKJJ0p9Pp9H+NqcOXMcr1u3bu20b+DAgZKkU6dOae7cuR7PT0lJ0XfffSdJ6t27t9O4cqBQUlKSDMMo9s+5Rc/JycmO7YW/7PN8PA8GgCLt27fPkGRIMm6//fZSnbNr1y4jNDTUkGS0b9/eyMrKctqflZVltG/f3pBkhIWFGb/99psXWo5Al5uba/Tt29fx+XvooYcu6Dp8HlEeEhMTjezs7GKPef311x2f14SEBMNutzvtT09PN+Li4gxJRsOGDY2jR4867bfb7Ub//v0d10hOTi7vt4EgMn78+BI/SzwfS4eaBeAcK1as0O7dux0/Hz161PF69+7dbtMBjhgxwu0azZs31+OPP64JEyZo/fr16ty5s5544gk1adJEe/bs0SuvvKJNmzZJkh5//HE1a9bMK+8Fge3WW2/V0qVLJUm9evXSqFGjnAr2XEVERKh58+Zu2/k8ojw8//zzevTRR3XjjTeqS5cuatKkiSpWrKjTp09r69at+uyzz7Ry5UpJZz+LH3zwgUJDQ52uUbVqVb3yyiu65557dODAAV1xxRV65pln1Lp1ax06dEhvvPGGkpOTJZ39/Pfo0cPXbxNBhudjKfk7rQBmcvvttzu+iSjNn6Lk5+cbd9xxR7Hnjho1ysjPz/fhu0MgOZ/Pof7/N7VF4fOIsmrYsGGpPofx8fHG0qVLi73Wc889Z9hstiKv0a9fvxJ7MYCSlKZnwTB4PpYGNQuAF4SEhGjatGlatGiRBgwYoLp16yoiIkJ169bVgAED9PXXX+vDDz9USAj/F4T38XlEWX3zzTd67bXXdMMNN+jSSy9VrVq1FBYWpkqVKqlJkya68cYblZiYqF27dumqq64q9lovvPCCVqxYoSFDhqh+/fqKiIhQzZo1ddVVV2nGjBlatGiR04JZgDfxfCyZzTAMw9+NAAAAAGA+wRuTAAAAABSLsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAAAAA8IiwAAAAAMAjwgIAAAAAjwgLAAAAADwiLAAAAADwiLAAAAAAwCPCAgAAAACPCAsAAAAAPCIsAAAAAPCIsAAAAADAI8ICAAAAAI8ICwAAAAA8IiwAACzt+eefl81mk81m83dTACDgEBYAIAjt37/f8Qt0Wf4AAKyNsAAA8KmkpCRH2Ni/f7+/m2Npy5Ytc/xdL1u2zN/NARCAwvzdAACA79WrV09bt24tcn/r1q0lSe3bt1diYqKvmgUAMBnCAgAEofDwcF1yySUlHhcTE1Oq4wAA1sQwJAAAAAAeERYAAOetoKBAn376qfr166fatWsrIiJCNWrUUM+ePTV58mTl5eW5nVM4fn7kyJGObQkJCW5F065j61evXq1nn31WPXr0cNwrNjZWrVq10r333qvt27d7++06nD59Wq+99pp69erl1JY2bdrogQce0MqVK4s898iRI3r22WfVpk0bVa5cWVFRUWrUqJGGDRumFStWlHjvH374QbfeeqsSEhJUoUIFRUdHq2HDhrryyiv12GOP6YcffnAcW1jA3rNnT8e2nj17uv1dJyUllenvA0AQMAAAcCHJkGR0797dbV96errRuXNnxzGe/rRs2dLYv3+/03nJycnFnlP4Jzk52XFOYmJiiceHhoYa7777bpHvZfz48Y5jy+Lbb781qlevXmJ7PPnmm2+M2NjYYs+7//77jfz8fI/njx07tsT7VqtWzXH8vn37SvV3nZiYWKa/EwDWR80CAKDU8vPz9fe//12rVq2SJHXv3l1jxoxRQkKCDh06pOnTp2vevHnasWOHevfurc2bN6tixYqSpA4dOmjr1q2aP3++nn32WUnSN998o7p16zrdIyEhwfHabrerSpUqGjBggLp166ZmzZopJiZGhw4d0saNG/XWW2/p6NGjGjNmjFq0aKFevXp55X0nJyfr2muvld1uV2hoqIYNG6YBAwaoQYMGysnJ0fbt27V48WItWLDA7dzNmzerf//+ysvLU3h4uMaMGaPrr79eMTEx2rRpkyZMmKB9+/bp3XffVUxMjF555RWn8xcuXKg33nhDknTppZfq3nvvVcuWLRUXF6cTJ07o119/1Xfffae1a9c6ziksYF+3bp3uuOMOSdL06dPVoUMHp2vHx8eX898UAMvxd1oBAJiPiuhZeOeddxz7hg8fbhQUFLid+/TTTzuOGTdunNv+c3sL9u3bV2w7UlJSjMzMzCL3nzhxwrj00ksNSUaXLl08HlPWnoXs7Gyjbt26hiQjOjraqefD1R9//OG2rUOHDo4ekG+++cZt/7Fjx4xWrVoZkoyQkBBj27ZtTvuHDRtmSDIaNmxonD59ush7p6enu207tzenuHYDQFGoWQAAlNq7774rSapRo4beeecdjwuzvfDCC2rRooUkaerUqcrNzb3g+9WrV0/R0dFF7o+Li9O//vUvSdKKFSuUnp5+wfcqyscff6xDhw5Jkl566SX16NGjyGPr16/v9PPatWu1bt06SdJdd92lvn37up1TpUoVffDBB5LO1oJMnjzZaf/hw4clSW3btnX00nhStWrVkt8MAJwnwgIAoFQOHTqkHTt2SJJuvvlmVapUyeNxYWFhjiLm48ePa+PGjeXWhszMTO3fv1+//vqrtm3bpm3btik8PNyxf8uWLeV2r0ILFy6UdHYa2bvuuuu8zv3uu+8cr0eNGlXkcZ07d1bLli3dzpGkOnXqSJJ+/PFH7dmz57zuDwBlRVgAAJTKtm3bHK+vuOKKYo89d/+5512Io0eP6umnn9ZFF12kSpUqKSEhQZdccolat26t1q1b67rrrnM6trxt2rRJktSuXbtiezk8KXzvERERuvzyy4s9tvDv7Pfff3eaTWr48OGSpPT0dF1yySUaPHiwEhMTtXv37vNqCwBcCMICAKBUjh075nhds2bNYo+tXbu2x/PO14YNG9SiRQu9/PLL+u2332QYRrHHZ2dnX/C9ilIYQAq/4T8fhe+9atWqCgsrfk6Rwr8zwzB0/Phxx/bevXvrnXfeUYUKFZSTk6NZs2bpjjvuULNmzRQfH6977rnHKz0qACARFgAAF8BTrUJ5y8vL080336z09HSFh4frkUce0fLly5WamqqcnBwZhiHDMJyG5pQUJvylrH9f999/v/bv369JkyapX79+iouLkyT9+eefmjJlitq0aeOYYQoAyhNhAQBQKucW0P7111/FHltYlOt63vn44YcftHfvXknS5MmT9dprr6lbt26qXbu2IiMjHceVpeeiNKpXry5JSk1NPe9zC997enq67HZ7sccW/p3ZbDZVqVLFbX/NmjU1duxYLVq0SMeOHdOGDRv07LPPqnLlyjIMQy+++KLmz59/3m0EgOIQFgAApXLJJZc4Xq9Zs6bYY8+d8//c86TSf8v+66+/Ol7fcsstRR63fv36Ul3vQrVt29Zxn6ysrPM6t/C95+XlafPmzcUeW/h31qxZM0VERBR7bEhIiNq2bat///vf+v777x3bZ8+e7XScL3qAAFgbYQEAUCp169Z1zNgze/ZsZWRkeDwuPz9fSUlJks5OC1r4y3ahqKgox+viplU995v4zMxMj8cUFBRo6tSppWr/herfv78kKSsryzHFaWn16dPH8Xr69OlFHrdq1Spt377d7ZzSaNu2raMnwrXAu7R/1wBQFMICAKDU7r//fknSkSNH9OCDD3o85oUXXnD84nvXXXc5DRmSnAuFi5sKtFmzZo7XheHD1VNPPVWuU7N6MnToUNWrV0+S9Mwzz2j58uVFHpuSkuL0c8eOHdW+fXtJZ9ecOLcXoNDJkyc1evRoSWd7DO69916n/bNmzSq2cHv9+vWOguhzV7+WSv93DQBFsRlmrQYDAPhN4fCV7t27a9myZY7t+fn56tq1q1atWiVJ6tWrl+677z4lJCQoNTVV06dP19y5cyVJTZo00ebNm90WEjt9+rRq1qypnJwctW3bVhMmTFDDhg0VEnL2+6t69eqpQoUKyszMVOPGjZWWlqbQ0FDdeeedGjRokKpXr67du3c7fvnu3LmzVq5cKUlKTEzUiBEjnO73/PPP64UXXpB04QXQycnJ6tu3r+x2u8LCwjRs2DANHDhQ8fHxys3N1c6dO/X111/rq6++cvsGf/PmzbriiiuUl5eniIgIPfDAA+rfv79iYmK0adMmTZgwwVGbMW7cOL3yyitO5zdq1EgnT57UgAED1K1bNzVv3lwxMTFKT0/XihUr9Pbbb+vYsWMKDQ3V6tWrHeGkUP369ZWSkqKEhAS98cYbuuiiixQaGipJqlWrVpHrZQCAJMlva0cDAExLkiHJ6N69u9u+9PR0o3Pnzo5jPP1p2bKlsX///iKvP27cuCLPTU5Odhy3ZMkSIyoqqshje/ToYWzbts3xc2Jiotu9xo8f79hfFkuWLDGqVKlS7Psu6h7ffPONERsbW+x5999/v5Gfn+92bsOGDUu8Z2RkpMf3bhiGMXny5CLPK+ocACjEMCQAwHmpWrWqfvzxR3388ce65pprVKtWLYWHh6tatWrq0aOH3nnnHW3evFkNGzYs8hoTJkzQ1KlT1bVrV1WtWtXxTberq6++WuvXr9fQoUNVt25dhYeHq0aNGurevbs++OADff/994qJifHWW3Vry969e/XSSy+pU6dOqlatmkJDQxUbG6u2bdtq7NixToXd5+rbt692796tp59+WpdffrliY2MVGRmpBg0a6LbbbtNPP/2kd955x9G7cq7k5GS9+eabuvHGG9W6dWvVqFFDYWFhio2NVZs2bfTYY49p+/btbj0qhe699159+eWX6tu3r2rWrFnieg8AcC6GIQEAAADwiJ4FAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB4RFgAAAAB4RFgAAAAA4BFhAQAAAIBHhAUAAAAAHhEWAAAAAHhEWAAAAADgEWEBAAAAgEeEBQAAAAAeERYAAAAAeERYAAAAAOARYQEAAACAR4QFAAAAAB79PyHokMOGCn9oAAAAAElFTkSuQmCC" + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwwAAAJECAYAAAC7A6POAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAB7CAAAewgFu0HU+AAB19UlEQVR4nO3dd3wUdf7H8fem91BC770LoakUqaKCCOhhl6qnnqCeKArngXo/+2EXFUSwnCJ4CiIWFAEBUXqTJlUgCSGhhPRsdn5/cFmzu9kUyO7OJq/n48GD2ZnvzHyWMWY/+/kWi2EYhgAAAACgCAG+DgAAAACAeZEwAAAAAHCLhAEAAACAWyQMAAAAANwiYQAAAADgFgkDAAAAALdIGAAAAAC4RcIAAAAAwC0SBgAAAABukTAAAAAAcIuEAQAAAIBbJAwAAAAA3CJhAAAAAOAWCQMAAAAAt0gYAAAAALhFwgAAAADALRIGAAAAAG6RMAAAAABwi4QBAAAAgFskDAAAeFjfvn1lsVhksVi0cuVKX4cDAGVCwgAAKJPCH36L+hMQEKDo6Gg1adJEw4cP1+zZs3Xu3Dlfhw0AuEAkDACAcmUYhtLT03X48GEtXrxYf/3rX9WiRQt9+eWXvg4NAHABgnwdAADAf3Xr1k3du3d32Gez2XTmzBlt27ZNu3btkiSdOHFC119/vRYtWqRrr73WF6ECAC4QCQMA4IINHjxYTzzxhNvja9eu1c0336xjx44pPz9f99xzjw4dOqTg4GDvBWkCjFsA4M/okgQA8JiePXtq4cKF9tfHjx/nwzMA+BkSBgCAR1122WVq0qSJ/XVBNyUAgH8gYQAAeFydOnXs2xkZGS7HDx8+bJ9lqXHjxvb9a9as0Z133qnWrVsrNjZWFotFDz74oMO5NptNq1ev1rRp0zRo0CA1bNhQERERCg0NVZ06ddS/f389/fTTSklJKVWshWd8KrB37149+OCDatOmjaKiohQTE6OOHTtqypQppbpuaaZVHTNmjL3NvHnzJEmZmZmaOXOmevXqpVq1aik0NFQNGjTQLbfcorVr15bq/QDAxWIMAwDA45KSkuzbtWvXLrF9bm6u7r//fr3zzjvFtsvLy1OTJk10/Phxt/dNSkrSihUr9Oyzz+rtt9/W7bffXqbY3377bT344IPKyclx2L99+3Zt375ds2fP1rfffquuXbuW6bol2bVrl/7yl79o9+7dDvuPHTum+fPna/78+Zo2bZqefPLJcr0vADgjYQAAeNTGjRt18OBB++vevXuXeM7f//53e7LQoUMHdezYUcHBwdq3b58CAv4sjufn59uThaioKLVr105NmzZVTEyM8vLydOzYMf3yyy9KS0tTRkaG7rjjDgUHB+umm24qVezz5s3TvffeK0lq1aqVunbtqvDwcO3Zs0dr166VYRhKTU3Vddddp927dys2NrbU/y7FSUhI0MCBA5WYmKgqVaqod+/eql27tlJSUvTjjz/q7NmzkqSnnnpKbdu2LfX7AYALYgAAUAZ9+vQxJBmSjOnTpxfbdv369Ubjxo3t7UeMGFFku0OHDtnbBAYGGpKMBg0aGD/99JNL2+zsbPt2Tk6OMXbsWGPFihVGbm5ukdfOzs42XnjhBSMoKMiQZFSpUsU4d+6c25gL4pBkhIaGGjVq1DC++eYbl3arVq0yYmJi7G2ffPJJt9cs/G+2YsWKItuMHj3a4b6SjEcffdTIyMhwaJeammr079/f3rZp06aGzWZze28AuFhUGAAAF+zrr7926cNvs9l09uxZbd++XTt37rTvHzFihD766KMSr5mfn6+IiAj98MMPatmypcvx0NBQ+3ZISIjee++9Yq8XGhqqRx55RDabTY899pjOnDmjDz/80F45KMkPP/ygSy65xGX/FVdcoWeeeUYTJkyQJH3yySeaNm1aqa5ZkpycHE2ZMkXPPPOMy7Fq1arp448/VrNmzZSRkaGDBw9q/fr1uvTSS8vl3gDgjIQBAHDBNmzYoA0bNhTbpk6dOpo5c6aGDx9e6utOmDChyGThYowdO1aPPfaYpPNJQGkShr/+9a9FJgsFRo0apQcffFBWq1V79+5VWlqaYmJiLjrWGjVqFJt81KpVS0OGDNGCBQskiYQBgEeRMAAAPCoxMVE33HCDbr31Vr322muqWrVqiefcfPPNZb6PzWbTpk2btHXrVh07dkxpaWnKy8srsu3WrVtLdc2RI0cWezw6OlrNmjXT3r17ZRiGjhw5og4dOpQ1dBdDhw5VWFhYsW3i4+PtCcPhw4cv+p4A4A4JAwDggk2fPr3IlZ4zMjJ0+PBhffPNN3rhhRd08uRJffTRR9qyZYtWr15dbNIQHBxcpg/dVqtVr732ml5++WUdO3asVOeUdorV0sRRvXp1+3ZaWlqprmvW+wJAUViHAQBQ7iIjI9WuXTs9/PDD2rJli+rVqydJ+u233/TQQw8Ve27VqlUVFFS677NycnI0ZMgQTZo0qdTJgiSdO3euVO1KM+tRcHCwfdtdRaOsfHVfACgKCQMAwKPq1aun6dOn219/9NFHDusyOAsPDy/1tZ988kktW7ZM0vkF12666SYtWLBAu3fv1tmzZ5WbmyvDMOx/ChTeLk7hxdu8yVf3BYCi0CUJAOBxV111lX3barVq1apVF712QE5Ojl5//XX763nz5mnUqFFu25e2qgAAcESFAQDgcXXq1HF4feTIkYu+5vr165Weni5JateuXbHJQnndEwAqIxIGAIDHZWZmOrwuvFrzhUpISLBvl2aQ8E8//XTR9wSAyoiEAQDgcZs3b3Z4XTAI+mIUTjqcExJnNptNs2bNuuh7AkBlRMIAAPC4l19+2b5tsVjUv3//i75m06ZN7durVq3S2bNn3bZ98cUXtW3btou+JwBURiQMAACPOXPmjO6++24tWbLEvu/WW29VrVq1Lvra8fHx9krF2bNnNXLkSIduStL5gdHTpk3TY489psjIyIu+JwBURsySBAC4YF9//XWRi6BlZmbq8OHD+uWXX5SVlWXf37JlS7300kvlcu+AgAD961//0rhx4yRJ33//vVq2bKkePXqoUaNGSk1N1cqVK3X69GlJ0qxZs3TbbbeVy70BoDIhYQAAXLANGzZow4YNpWp73XXX6Z133lHNmjXL7f5jx47V/v379cwzz0g6v8L0999/79AmLCxMr7zyim699VYSBgC4ACQMAIByFxoaqtjYWDVv3lyXXXaZbr31VnXp0sUj93r66ad1zTXX6I033tCaNWt08uRJRUdHq379+rr66qs1fvx4tWjRwiP3BoDKwGKUdrlLAAAAAJUOg54BAAAAuEXCAAAAAMAtEgYAAAAAbpEwAAAAAHCLhAEAAACAWyQMAAAAANwiYQAAAADgFgkDAAAAALdIGAAAAAC4RcIAAAAAwC0SBgAAAABukTAAAAAAcIuEAQAAAIBbJAwAAAAA3ArydQAwr+zsbO3YsUOSVKNGDQUF8Z8LAACAWVmtVp08eVKS1KFDB4WFhZXLdfkECLd27Nih7t27+zoMAAAAlNH69evVrVu3crkWXZIAAAAAuEWFAW7VqFHDvr1+/XrVqVPHh9EAAACgOImJifbeIYU/x10sEga4VXjMQp06dVS/fn0fRgMAAIDSKs+xp3RJAgAAAOAWCYOfOHLkiCZNmqTWrVsrMjJS1apVU7du3fTiiy8qMzPT1+EBAACggqJLkh9YsmSJbr/9dqWlpdn3ZWZmauPGjdq4caPeffddLV26VM2bN/dhlAAAAKiIqDCY3JYtW3TTTTcpLS1NUVFRevrpp/Xzzz9r+fLluuuuuyRJ+/bt05AhQ3Tu3DkfRwsAAICKhgqDyT3wwAPKyspSUFCQli1bpssvv9x+rH///mrRooUmT56sffv2acaMGXriiSd8FywAAAAqHCoMJrZ+/XqtXr1akjR+/HiHZKHApEmT1KZNG0nSq6++qry8PK/GCAAAgIqNhMHEFi1aZN8eO3ZskW0CAgI0atQoSdKZM2e0YsUKb4QGAACASoKEwcTWrFkjSYqMjFSXLl3ctuvTp499e+3atR6PCwAAAJUHCYOJ7d69W5LUvHnzYhffaN26tcs5AAAAQHlg0LNJZWdnKyUlRZJKXGG5atWqioyMVEZGho4ePVrqexw7dqzY44mJiaW+FgAAAComEgaTKjxFalRUVIntCxKG9PT0Ut+jQYMGFxQbAAAAKg+6JJlUdna2fTskJKTE9qGhoZKkrKwsj8UEAACAyocKg0mFhYXZt3Nzc0tsn5OTI0kKDw8v9T1K6r6UmJio7t27l/p6AAAAqHhIGEwqOjravl2abkYZGRmSStd9qUBJYyMAAOZkGIbybYasNkN5+TZZ8w3l2f73d75NefmGJMPjcVhthjJz85WZk6+MXKsyc63KyMm3/52Vl6+MHKsyc//822qzeTwuwIxa1IzWv4a393UYF4SEwaTCwsJUvXp1paamljg4+fTp0/aEgXEJAHDx8m2GTmXkKjUjR6npuUpJz1FKeq5S03OUkv6/fRnnX2fn5Xs8Hpshe2JgtRUkBAD8SY7Vf5NlEgYTa9u2rVavXq39+/fLarW6nVp1z5499u2CVZ8BoKzOZObq5e/3aU/SORmGZMj439+SzfhzW4YhW+HjFeiza16+TacycnUqM7dCvS8AuBgkDCbWq1cvrV69WhkZGdq0aZMuvfTSItutWrXKvt2zZ09vhQegArHZDN31wUZtOHza16EAAEyGhMHEhg8frmeffVaSNHfu3CITBpvNpg8++ECSVKVKFfXr18+rMQKoGD7bdIxkARckIiRQESFBigz9398hgQoPCVRkSJAiQp3+DglUcCATNKJyqhEd6usQLhgJg4l1795dvXv31urVqzVnzhyNHj1al19+uUObGTNm2Fd3fuCBBxQcHOyLUAH4sbOZeXr+2z0lN6ykqkYEq3pUqOKiQs7/HRmiuKhQVY8KVfWoEEWFBsni6SAsUnBggIICLOf/DrQoKCBAwYEWBQUGKDjg/N9BgRYFB5z/O9Di8ahksUgWL9wHgG+RMJjcq6++qp49eyorK0uDBg3S1KlT1a9fP2VlZWn+/PmaNWuWJKlly5aaNGmSj6MF4I9e+n6vUjMcp2++v39z1YwJO/+BUJb//S0F/G/DovMfFAMs5z80VhQBFouqRYaoemSo4qJDVC0iREFm/0bcZpOOrJV+/07KPlu+1w6LlQb9X/leE4DfIWEwufj4eH366ae6/fbblZaWpqlTp7q0admypZYuXeowFSsAlMZvCWf14S9HHPYNaltLDw1q5aOIUGqnD0tbP5G2fSyd+cMz94iuS8IAgITBHwwdOlTbt2/Xq6++qqVLl+rYsWMKCQlR8+bNNXLkSE2YMEERERG+DhOAnzEMQ9MX/yZbodmAQoMC9M9r2/ouKBQvJ13a/aW05T/SkTW+jgZAJUHC4CcaNWqkl156SS+99JKvQwFQQXy++bg2HnEc6Hxfv+ZqUI0vIEzFZpP++Fna+rH02yIpL8PXEQGoZEgYAKASSsvO07PfOA50blgtQn+9oqmPIoKL04elbfPPJwpnjpTYXDXaSE16S5bA8oshLKb8rgXAb5EwAEAl9PL3+5SSnuOwb/rQtgoLLscPmxfjj1+kH/9PSjvu60h8w7CdTxhKElZF6jBS6nSrVDe+Yo1AB2AaJAwAUMnsSUrTB+scv7Ee0LqmBrSp5aOInOTnSR/fJGWf8XUk5mQJkJpfeT5JaHWNFOS/c7sD8A8kDABQiRiGoWmLf1N+oZHOIUEBmj60nQ+jcnLqIMlCUeJaSfG3SZfcJEXX9nU0ACoREgYAqES+3Jag9YdOOey7p08zNaxuooHOeZm+jsA8wmILdTnqTJcjAD5BwgAAlcS57Dw9vXS3w776VcP1t77NfBSRG3nZjq+DI6XrZ/kmFl8Ki5Hqd5eCw3wdCYBKjoQBACqJ15b/ruRzjgOd/3mtiQY6F7BmOb4OjZLaXOubWAAAMvl69wCA8vD7iXOau/aww74+LWtoUFuTDHQuzLnCEMQ37ADgSyQMAFDBFQx0thYe6BwYoCeuayeLGfvEO1cYgsN9EwcAQBIJAwBUeEt3JGrdwVSHfXdd0URN4iJ9FFEJXCoMTBsKAL5EwgAAFVhGjlX/95XjQOe6sWG6r19zH0VUCs4VhiAqDADgSyQMAFCBvf7jfiWlOX5j/89r2yoixMRzXlgdB2YzSxAA+BYJAwBUUAdOpmvOmoMO+3o1j9PV7U2+6FceFQYAMBMTf8UEACiwOzFN3+86oTOZeaU+59dDqcrL/3Ogc3CgxbwDnQuzOq/DQIUBAHyJhAEATCo7L19f70jUf379Q5uOnL7o643r1UTNa0aVQ2QeRoUBAEyFhAEATOZQSoY+/vWIFm46VqaKQnFqx4Tp/v4tyuVaHkeFAQBMhYQBAEwgL9+mH3ad0H9+/UNr9qeU67UtFumJ69opMtRP/pfvMq0qFQYA8CU/+e0BABVTwpkszV//h+ZvOKrkcznFtq0ZHareLWooKKD0YxBCgwN0Zdta6t2ixsWG6j0uC7dRYQAAXyJhAAAfWLs/RXPXHtKPe5JVaAHmIvVqHqfbL2uoAW1qKTiwEkxuR4UBAEyFhAEAvGzu2kN6csmuYttUjQjWyK4NdEv3huZdkdlTqDAAgKmQMACAFxmGobdXHXB7vGujqrrtsoa6pn0dhQUHejEyE3GpMJAwAIAvkTAAgBcdO52lE2mOYxWiQoM0Ir6ebr20odrUifFRZCbiUmGgSxIA+BIJAwB40eY/HNdTqBYZotWT+/nPDEbeQIUBAEylEoyeAwDzcF6ArXPDqiQLzpwrDCQMAOBTJAwA4EXOCUOXRlV9FImJOVcYGPQMAD5FwgAAXpKRY9XuxDSHfSQMRXCpMDCGAQB8iYQBALxk29EzDmsuBAVYdEn9WN8FZFZWpwXsqDAAgE+RMACAlzh3R2pXL7byTp3qjmFIVhZuAwAzIWEAAC/Z9IfzgOcqvgnEzJyTBYkKAwD4GAkDAHiBzWZoMwOeS5aX5bqPCgMA+BQJAwB4wYGT6UrLtjrsI2EoAhUGADAdEgYA8ALn8Qt1Y8NUJ5Zvzl1QYQAA0yFhAAAvcFmwjepC0VwqDBYpKNQnoQAAziNhAAAvcB7wTHckN5wXbQsKkywW38QCAJBEwgAAHnc6I1cHT2Y47CNhcMN50TbGLwCAz5EwAICHbTnqWF0ICw5QmzoxPorG5FwqDIxfAABfI2EAAA9zHr/QsX4VBQfyv98iUWEAANPhNxYAeJhzwkB3pGJQYQAA0yFhAAAPysu3advRsw77SBiK4VxhYIYkAPA5EgYA8KA9ieeUlZfvsC++IQmDW84VhmAqDADgayQMAOBBm46ccnjdtEakqkWG+CgaP+C8DkMQYxgAwNdIGADAgzb9ccbhdReqC8VzThioMACAz5EwAIAHbWbAc9nkOY9hoMIAAL5GwgAAHpJ4NkvHzzh+AO5MwlA8lwoDCQMA+BoJAwB4yOYjZxxeR4cFqXmNKN8E4y9cKgx0SQIAXyNhAAAPcV5/oXPDqgoIsPgoGj9BhQEATIeEAQA8ZNMfjF8oMyoMAGA6JAwA4AHZefnalcCCbWVGhQEATIeEAQA8YMfxs8rLN+yvAyxSxwZVfBeQv6DCAACmQ8IAAB7gPH6hde0YRYUG+SgaP0KFAQBMh4QBADzAOWGgO1IpUWEAANMhYQCAcmYYBgu2XSgqDABgOiQMAFDOjqRmKjUj12EfCUMp5TklDKz0DAA+R8IAAOXMuTtSjehQ1a9K15pSsTp3SSJhAABfI2EAgHLmsv5Cw6qyWFiwrVScKwzBJFoA4GskDABQzhi/cBGcxzBQYQAAnyNhAIBylJadp70nzjns60zCUDq2fMmW57iPCgMA+BwJAwCUo61/nJHx53ptCgkMUPt6Mb4LyJ84T6kqUWEAABMgYQCAcuQ84Ll9vRiFBgX6KBo/49wdSaLCAAAmQMIAAOVos/OAZ7ojlR4VBgAwJRIGACgn+TZDW/8447CPhKEMqDAAgCmRMABAOfk9+ZzO5Vgd9nVuSMJQas4VBkugFBjsm1gAAHYkDABQTpzHLzSoFq6aMXSpKTXnCgPVBQAwBRIGACgnzglDF6oLZeNcYWD8AgCYAgkDAJQTFmy7SFQYAMCUSBgAoBykpOfocGqmwz4WbCsjKgwAYEokDABQDpyrC5EhgWpVK9pH0fgplwoDCQMAmAEJAwCUg01O6y90alhFQYH8L7ZMqDAAgCnx2wwAyoHL+AUGPJedc4WBhAEATIGEAQAuUq7Vpm3HzjrsY/zCBWDQMwCYEgkDAFyk3xLOKtdqc9gXT4Wh7PKoMACAGZEwAMBFcl5/oWWtKMWGs0JxmVmdxjBQYQAAUyBhAICLtPkP1l8oF1QYAMCUSBgA4CIYhuFSYaA70gWiwgAApkTCAAAXIeFstk6k5Tjso8JwgagwAIApkTAAwEVwri5UiQhW07hIH0Xj56gwAIApkTAAwEVY8/tJh9ddGlaVxWLxUTR+jgoDAJgSCYMHpKen66efftK///1v3XjjjWrSpIksFossFosaN25c5uvt3LlTd999t5o1a6bw8HDVqFFDvXv31ttvvy2r1Vr+bwBAqeTbDP2wO9lh3+XNqvsomgqAdRgAwJSCfB1ARTR06FCtXLmyXK41e/ZsTZgwQbm5ufZ92dnZWrNmjdasWaO5c+dq6dKliouLK5f7ASi9jYdP6VRGrsO+q9rV9lE0FUCeU5ckKgwAYApUGDzAMAz7drVq1TRo0CBFRUWV+Tpff/217rnnHuXm5qpWrVp67bXX9Ouvv+qbb77R9ddfL0lav369RowYofz8/HKLH0DpLNt1wuF1mzoxalAtwkfRVABUGADAlKgweMCtt96qu+++W926dVPz5s0lSY0bN1Z6enqpr5GXl6eJEyfKZrMpJiZGa9euVbNmzezHr776at13332aOXOm1qxZow8//FBjxowp77cCwA3DMPTdb0kO+65qV8tH0VQQVBgAwJSoMHjAX//6V91yyy32ZOFCfPHFFzp48KAkacqUKQ7JQoEXX3xRVatWtW8D8J7died07LTjB9xBbemOdFGcKwwkDABgCiQMJrVo0SL7trvKQUREhG688UZJ0q5du7Rv3z4vRAZAkpbtcqwuNKgWrjZ1on0UTQXhXGEIJmEAADMgYTCpNWvWSJJatWql2rXdf2vZp08f+/batWs9HheA8777zXH8wqC2tZlO9WK5VBgYwwAAZkDCYELp6ek6evSoJKl169bFti18fPfu3R6NC8B5R09landimsO+QW0Zv3BRDKOIQc9UGADADBj0bELHjh2zb9evX7/Ytg0aNLBvFyQZF3KfoiQmJpbpekBl4TzYuVpkiLo2ruajaCqI/DzJsDnuo8IAAKZAwmBC586ds2+XNB1rZGSkfbssszBJjskGgNJznk51YJuaCgygO9JFsWa57qPCAACmQJckE8rO/rMsHxISUmzb0NBQ+3ZWVhG/cAGUq9T0HG08fMphH4u1lYO8bNd9VBgAwBQqbYWhPAYnzp071yNrH4SF/fmtWuEVnouSk5Nj3w4PL9sv15K6MCUmJqp79+5luiZQ0S3fnSzbn2szKiIkUD2bs9L6RaPCAACmVWkTBjOLjv5zasaSuhllZGTYt8u6mnRJ4yMAuHIev9C3VQ2FBQf6KJoKhAoDAJhWpU0YymNGoTp16pRDJK7q1atn3y5pYHLhKgFjEgDPysixavX+FId9LNZWTpwrDIEhUgC9ZgHADCptwlDSdKW+FB0drQYNGujo0aPas2dPsW0LH2/Tpo2nQwMqtZ/2nVSu9c+ZfIICLOrXuqYPI6pAnCsMVBcAwDT4+sakevXqJUnau3evkpKS3LZbtWqVfbtnz54ejwuozJy7I13erLpiw4N9FE0F41xhYPwCAJgGCYNJDR8+3L49b968IttkZmZqwYIFkqS2bduqZcuWXogMqJzy8m1avifZYR+LtZUjlwoDCQMAmAUJg0mNGDFCTZs2lSQ9++yzOnDggEubRx55RKdPn7ZvA/CcXw6m6ly21WHflYxfKD8uFQa6JAGAWVTaMQyetH//fq1Zs8ZhX8FsR+np6S4Vg6uvvlq1azt+8AgODtbrr7+uoUOHKi0tTT179tTjjz+u7t276/Tp05o9e7b++9//SjrffemOO+7w3BsCoGW/OS7W1rFBFdWO5VvwckOFAQBMi4TBA9asWaOxY8cWeSw1NdXl2IoVK1wSBkkaPHiw3n77bU2YMEEnTpzQxIkTXdp0795dX3zxhQIDmdYR8BSbzdCyXY7jF65qR3ekcuVcYSBhAADToEuSyd11113atGmT7rrrLjVt2lRhYWGqXr26evXqpbfeektr165VXByLRgGetP34WZ1Iy3HYx3Sq5cy5wsCgZwAwDSoMHjBmzJhyXQG6ffv2mjVrVrldD0DZOM+O1KxGpJrXLNtCiSiBS4WBMQwAYBZUGACgBMucEoZB7agulDurYwWHCgMAmAcJAwAUY39yug6czHDYx3SqHpBHhQEAzIqEAQCK4TzYuVZMqDrWr+KbYCoyK2MYAMCsSBgAoBjO06le2baWAgIsPoqmAnOZVpUKAwCYBQkDALiRdDZbW4+ecdh3FeMXPMNl4TYqDABgFiQMAODG97sdqwvRYUG6tEl1H0VTwVFhAADTImEAADecZ0ca0LqmQoL436ZHUGEAANPiNx8AFOFsVp7WHUh12Md0qh7kUmEgYQAAsyBhAIAirNiTLKvNsL8OCQpQn5Y1fBhRBedSYaBLEgCYBQkDABTBeTrV3s3jFBka5KNoKgEqDABgWiQMAOAkOy9fK/eedNg3qB2LtXkUFQYAMC0SBgBwsnZ/ijJz8+2vAyzSwDYkDB7lUmEI9U0cAAAXJAwA4MR5sbaujaqpehQfYD3KucLAtKoAYBokDABQSL7N0A9O6y/QHckLnCsMTKsKAKbhsxF8aWlpOnfunPLz80ts27BhQy9EBADSpiOnlZqR67BvUFumU/Uow5Dycxz3UWEAANPwasLw/fffa+bMmVqzZo1OnTpVqnMsFousVquHIwNQUWXn5Wvn8bMqNENqseav/8Phdeva0WpYPcIDkcHOmu26jwoDAJiG1xKG+++/X2+++aYkyTBK+ZsbAC7C+kOnNHbuemXkllzJdOcqFmvzvLws131UGADANLySMHz88cd64403JElhYWEaPny4unTpomrVqikggGEUADzjX1/tuqhkQWL8gldQYQAAU/NKwvDOO+9Ikho0aKAff/xRzZo188ZtAVRiKek52nH87EVdo3XtaLWtE1NOEcEtKgwAYGpeSRi2b98ui8Wi6dOnkywA8IpfDqY6vLZYpJiw4FKf36FerP55bVtZLJbyDg3OXCoMFtZhAAAT8UrCkJeXJ0mKj4/3xu0AQD8fcEwY+rSsoXlju/soGhTLZdG2sPMZHgDAFLwygKBx48aSpPT0dG/cDgC0zilh6NGsuo8iQYmcF21j/AIAmIpXEobrr79ekrR8+XJv3A5AJZdwJkuHUjIc9vVoFuejaFAilwoD4xcAwEy8kjBMmjRJDRs21CuvvKI9e/Z445YAKjHn7kix4cEMXjYzKgwAYGpeSRhiY2P13XffqVatWurRo4dmzpyp06dPe+PWACqhnw+kOLy+vGl1BQTQJ960qDAAgKl5ZdBz06ZNJUmZmZk6c+aMJk6cqPvvv19xcXGKiCh+BVWLxaIDBw54I0wAFYBhGK7jF5ozfsHUqDAAgKl5JWE4fPiww2vDMGQYhpKTk0s8lykNAZTFoZQMJZ51/Maa8QsmV9QsSQAA0/BKwjB69Ghv3AYAXMYv1IwOVbMakT6KBqXiXGEgYQAAU/FKwjB37lxv3AYAipxOlUqlyTlXGIIZwwAAZuKVQc8A4A02m6F1B50TBrojmZ7zSs9UGADAVEgYAFQYe5LO6VRGrsM+Bjz7AeeEgUHPAGAqXumS5CwrK0ubNm1SUlKSMjMzNXz4cMXEMEc6gIvjPJ1qw2oRql+1+JnYYAJ5zmMY6JIEAGbi1YTh6NGjmjp1qhYuXKi8vDz7/q5du6pt27b213PmzNE777yj2NhYLVu2jP7HAEqlqPEL8ANUGADA1LzWJenXX39VfHy8Pv74Y+Xm5tqnVi3K0KFDtX37dv34449atmyZt0IE4Mes+Tb9euiUw77LSRj8AxUGADA1ryQMZ86c0bBhw3Tq1CnVrl1bM2fO1I4dO9y2r1mzpq655hpJ0tKlS70RIgA/t/34WaXnWB32MeDZT1BhAABT80qXpNdee03JycmKi4vTunXr1LBhwxLPGThwoBYvXqz169d7IUIA/s65O1LLWlGqER3qo2hQJlQYAMDUvFJhWLJkiSwWix566KFSJQuS1K5dO0nSgQMHPBkagArCecAz1QU/QoUBAEzNKwnD/v37JUlXXHFFqc+pWrWqJCktLc0jMQGoOLLz8rXx8GmHfYxf8CNUGADA1LySMGRnn//2KDg4uNTnZGRkSJLCw/nFAaB4m/84rRyrzf46wCJd1pSEwW9QYQAAU/NKwlCzZk1J0qFDh0p9ztatWyVJdevW9URIACoQ5/EL7evFKja89F9QwMfynFd65osiADATryQMl156qSTpm2++KVV7wzA0e/ZsWSwW9e7d25OhAagAfnZKGOiO5Geszl2SGKwOAGbilYThtttuk2EY+s9//mOvHBRn0qRJ2rZtmyRp9OjRHo4OgD9Lz7Fq29EzDvsY8OxnnCsMwVQYAMBMvJIwDBs2TP369ZPVatWAAQP01ltvKTk52X7carUqISFBCxcuVO/evfXqq6/KYrHo+uuvV48ePbwRIgA/teHQKVltfy4CGRxoUbfGVX0YEcrMpcLAGAYAMBOvrMMgSf/97381YMAAbdmyRRMmTNCECRNksVgkSfHx8Q5tDcPQZZddpnnz5nkrPAB+ynk61fgGVRUR4rX/teFi5Vslm+OCe1QYAMBcvFJhkKQqVapo3bp1mjJlimJiYmQYRpF/wsPDNXnyZK1cuVKRkZHeCg+An2L8gp9zniFJosIAACbj1a/hQkJC9PTTT2vq1KlatWqVNm7cqOTkZOXn56t69eqKj4/XwIEDFRsb682wAPip0xm52pXouFZLDxIG/1JUwkCFAQBMxSd1+8jISA0ePFiDBw/2xe0BVBC/HEyV8efwBYUFByi+IeMX/Irzom0SFQYAMBmvdUkCgPLm3B2pW+NqCgnif2t+hQoDAJieTyoMWVlZ2rRpk5KSkpSZmanhw4crJibGF6EA8GPOA56ZTtUPOVcYLIFSIIvuAYCZeDVhOHr0qKZOnaqFCxcqLy/Pvr9r165q27at/fWcOXP0zjvvKDY2VsuWLbPPpgQABU6kZevAyQyHfYxf8EPOFQaqCwBgOl6r3f/666+Kj4/Xxx9/rNzcXPusSEUZOnSotm/frh9//FHLli3zVogA/IhzdSE6LEjt6zFhgt9xrjAwfgEATMcrCcOZM2c0bNgwnTp1SrVr19bMmTO1Y8cOt+1r1qypa665RpK0dOlSb4QIwM/8vN9x/MJlTasrMIBqpN+hwgAApueVLkmvvfaakpOTFRcXp3Xr1qlhw4YlnjNw4EAtXrxY69ev90KEAPyJYRguA57pjuSnqDAAgOl5pcKwZMkSWSwWPfTQQ6VKFiSpXbt2kqQDBw54MjQAfujoqSwdP+P4QZMBz37KpcJAwgAAZuOVhGH//v2SpCuuuKLU51Sten4u9bS0tBJaAqhs1jqNX4iLClHLWlE+igYXxaXCQJckADAbryQM2dnnv0EKDi79VHkZGednPwkP55cHAEfO3ZEubxbHbGr+yrnCEBTqmzgAAG55JWGoWbOmJOnQoUOlPmfr1q2SpLp163oiJAB+yjAMrXNZf4HxC37LucLAoGcAMB2vJAyXXnqpJOmbb74pVXvDMDR79mxZLBb17t3bk6EB8DO/J6crJT3XYR8Jgx9zqTAwhgEAzMYrCcNtt90mwzD0n//8x145KM6kSZO0bds2SdLo0aM9HB0Af7J2v2N1oV6VcDWsFuGjaHDRmFYVAEzPKwnDsGHD1K9fP1mtVg0YMEBvvfWWkpOT7cetVqsSEhK0cOFC9e7dW6+++qosFouuv/569ejRwxshAvATRU2nyvgFP5ZHhQEAzM4r6zBI0n//+18NGDBAW7Zs0YQJEzRhwgT7L/n4+HiHtoZh6LLLLtO8efO8FR4AP5BvM/TLQaeEoTndkfyalTEMAGB2XqkwSFKVKlW0bt06TZkyRTExMTIMo8g/4eHhmjx5slauXKnIyEhvhQfAD/yWcFbnsq0O+1h/wc9RYQAA0/NahUGSQkJC9PTTT2vq1KlatWqVNm7cqOTkZOXn56t69eqKj4/XwIEDFRsb682wAPiJtfsdqwvNakSqVgwfMP0aFQYAMD2vJgwFIiMjNXjwYA0ePNgXtwfgp352mU6V6oLfo8IAAKbntS5JAHAxcq02bTh8ymEf06lWAMySBACm55MKQ1FOnDihr776SikpKWrSpImuvfZaRUQwVSKA87YePaPsPJv9tcUiXdaUhMHvOS/cRoUBAEzHKwnD7t27NX36dFksFr3zzjuqUqWKw/Evv/xSt956q7Ky/vzFUb9+fS1evFidOnXyRogATM55dqS2dWJUNTLER9Gg3FBhAADT80qXpEWLFumzzz5TQkKCS7KQnJys22+/XZmZmQ6zJR09elRDhw5Venq6N0IEYHJb/jjt8Lp7k2o+igTligoDAJieVxKG5cuXy2Kx6Nprr3U5NnPmTKWnpysoKEgvvfSStm3bphdeeEEBAQFKSEjQ7NmzvREiABMzDENbj55x2BffsKpvgkH5osIAAKbnlYThjz/+kOS6QJt0fkE3i8WiUaNG6cEHH1SHDh308MMPa/z48TIMQ19++aU3QgRgYkdSM3U6M89hX3yDKr4JBuWLCgMAmJ5XEobk5GRJUs2aNR32p6Sk6LfffpMk3XrrrQ7HrrvuOknSrl27vBAhADPbctSxO1JcVKjqV+Wb6ArBucJAwgAApuOVhKFgMHN2tuMvhjVr1kg6v6Bbr169HI7VqVNHknTmzBnPBwjA1Lb8ccbhdXzDKrJYLL4JBuXHMFwrDMEkDABgNl5JGKpVOz84saBrUoHly5dLkrp27aqQEMfZTqxWqyQpKirKCxECMDPn8Qud6I5UMeTnSTIc9wVROQIAs/FKwtCxY0dJ0scff2zfl5WVpYULF8pisah///4u5xw5ckSSVKtWLW+ECMCksvPytSshzWFffMMqvgkG5cua5bqPCgMAmI5XEoabb75ZhmFoyZIluvnmm/XGG29o0KBBSk5OlsVi0S233OJyzq+//ipJatSokTdCBGBSO4+fldX257fQARbpkvpVfBcQyk9etus+KgwAYDpeSRhGjRqlXr16yTAMLVy4UA888IB+/vlnSdLYsWPVunVrl3M+//xzWSwW9ejRwxshAjAp5/ELLWtFKyrUNIvU42JQYQAAv+CVhCEgIEDffPONHnroIdWvX19BQUFq0KCB/vnPf+qtt95yaf/VV1/p8OHDkqTBgwd7I0QAJuW6/kIVn8QBD6DCAAB+wWtf00VGRurf//63/v3vf5fYtmfPnjp06JAk/+ySdPjwYS1ZskQrV67U9u3bdfz4cdlsNsXFxalr1666+eab9Ze//EVBQaX759+5c6def/11/fDDD0pISFBUVJRat26t2267TXfeeWeprwP4I+cVnuMbsGBbheFcYQgMkQK88j0WAKAMTPlJs2rVqqpa1T8/FPzzn//U008/LcMwXI4dP35cx48f1+LFi/XSSy/ps88+U8OGDYu93uzZszVhwgTl5uba92VnZ2vNmjVas2aN5s6dq6VLlyouLq7c3wvgayfSspVw1vFbaCoMFYhzhYHqAgCYEl/llLPExEQZhqHIyEjdfvvtmjt3rtasWaONGzfqww8/VLdu3SRJGzZs0MCBA5Wenu72Wl9//bXuuece5ebmqlatWnrttdf066+/6ptvvtH1118vSVq/fr1GjBih/Px8r7w/wJucxy9EhwapWQ2mWq4wnCsMjF8AAFMyZYXBn1WvXl3PP/+87r33XkVHRzsc69Kli2655RbdeuutWrBggX7//Xe99NJLmjZtmst18vLyNHHiRNlsNsXExGjt2rVq1qyZ/fjVV1+t++67TzNnztSaNWv04YcfasyYMZ5+e4BXOa/w3LFBFQUEsGBbheFSYSBhAAAzosJQzp5//nlNnjzZJVkoEBgYqJkzZ9oXqvvss8+KbPfFF1/o4MGDkqQpU6Y4JAsFXnzxRXvXrRdffLE8wgdMpagVnlGBuFQY6JIEAGZEwuAD1atX1yWXXCJJOnDgQJFtFi1aZN92VzmIiIjQjTfeKEnatWuX9u3bV65xAr5kzbdpx7GzDvtIGCoYKgwA4BdIGHwkJydH0vmKQ1HWrFkjSWrVqpVq167t9jp9+vSxb69du7YcIwR8a++Jc8rKcxyb05EF2yoWKgwA4BdIGHwgOTlZu3fvliS1adPG5Xh6erqOHj0qSUUualdY4eMF1wQqAufuSI2qR6h6VKhvgoFnuFQYeL4AYEYMevaBF198UVarVZLsXYoKO3bsmH27fv36xV6rQYMG9u2CJKO0Ct+nKImJiWW6HlCeXMYvNKjikzjgQc4VBqZVBQBTImHwsl9//VWvvPKKpPPJwL333uvS5ty5c/btqKjip5CMjIy0bxc3RWtRCicbgNlsdZohqRMJQ8XjXGFgWlUAMCW6JHnRiRMn9Je//EVWq1UWi0Xvv/++IiIiXNplZ//5S7RgNiV3QkP/LOFnZWUV0xLwH2cz83TgZIbDvviG/rmYI4phZeE2APAHXqkwNGnSRAEBAfruu+/UvHnzUp3zxx9/qG/fvrJYLG5nEroYFsvFz+U+d+7cUq99cO7cOQ0ZMsTeDei5555T//79i2wbFvbnt2yFV3guSsHgaUkKDy/bL9uSujAlJiaqe/fuZbomUB62Hjvj8DokKEBt6sT4Jhh4jnPCQIUBAEzJKwnDkSNHZLFYSvzwW1heXp4OHz5cLh/sfS07O1vDhg3Tpk2bJEkPP/ywJk+e7LZ94TUcSupmlJHx57ewJXVfclbS+AjAV7b84dgdqUO9WIUEURCtcFwGPVNhAAAzqrRjGMpjRqE6deqU2MZqterGG2/UihUrJEl33nlniYus1atXz75d0sDkwlUCxiSgoth69IzDa8YvVFAu06pSYQAAMzJtwnD27PkFm4rq418eSpqutDzYbDbdcccdWrJkiSTppptu0jvvvFPiedHR0WrQoIGOHj2qPXv2FNu28PGipmgF/I1hGKzwXFlQYQAAv2DaGv9HH30kSWrUqJGPI7lwd999t+bPny9JGjp0qD766CMFBJTun7xXr16SpL179yopKcltu1WrVtm3e/bseRHRAuZwKCVDZ7PyHPYx4LmCosIAAH7BIxUGd4N5x44d6zANaFFycnJ08OBBJScny2KxaNCgQZ4I0eMeeughvfvuu5KkAQMGaOHChQoKKv0/9/Dhw/XJJ59IkubNm6fHHnvMpU1mZqYWLFggSWrbtq1atmxZDpEDvuVcXagRHaq6sXyQrJBcKgw8ZwAwI48kDCtXrpTFYpFhGPZ9hmFow4YNZbpO06ZNNWXKlPIOz+OeeOIJvfzyy5KkHj16aPHixQ7Tn5bGiBEj1LRpUx08eFDPPvusRo4cqWbNmjm0eeSRR3T69Gn7NlAROI9fiG9QpUJMfoAiuFQY6JIEAGbkkYThiiuucPgFv2rVKlksFnXp0qXYCoPFYlFYWJjq1KmjHj166Oabby6xImE2r7/+up588klJ5wcvv/DCCzp06FCx57Rq1UrBwcEO+4KDg/X6669r6NChSktLU8+ePfX444+re/fuOn36tGbPnq3//ve/ks53X7rjjjs884YAL9vitGAb3ZEqMCoMAOAXPFZhKKyg3/68efPUtm1bT9zSNAo+xEvS8ePH7WMRinPo0CE1btzYZf/gwYP19ttva8KECTpx4oQmTpzo0qZ79+764osvFBgYeFFxA2aQlZuv3YnnHPYx4LkCo8IAAH7BK7MkjRo1ShaLRVWr8k1hWd111126/PLL9dprr2n58uVKSEhQZGSk2rRpo9tuu0133nlnmcZGAGa2M+Gs8m1/dmUMsJxfgwEVFBUGAPALXvmkOW/ePG/cxhScqyvloX379po1a1a5XxcwG+cF21rVjlFkKAlxhUWFAQD8gql+Ex84cEApKSlq3LixatWq5etwAHgZ6y9UMi4VhrJNDgEA8A6vrMOQnJysmTNnaubMmfYF2Qrbv3+/unTpopYtW6pHjx6qV6+ebrjhBvsMQAAqB5eEgRWeKy6bTcrPcdzHwm0AYEpeSRg+//xzTZgwQa+++qpiYx37I+fk5Oiaa67R1q1bZRiGDMOQzWbTokWLNGzYMG+EB8AEEs9mKSnN8RtnKgwVmHOyILFwGwCYlFcShmXLlslisWjEiBEux+bNm6cDBw5Ikq677jq9+uqrGjp0qAzD0Nq1a/Xpp596I0QAPrbVqboQHRakpnFRvgkGnpeX5bqPCgMAmJJXEoa9e/dKki677DKXYx9//LGk86tDL1q0SBMnTtTixYs1cOBAGYah+fPneyNEAD62xWnBtk4NqigggAXbKixrtus+KgwAYEpeSRhOnjwpSapfv77D/qysLP3yyy+yWCz661//6nBs3LhxkqTNmzd7I0QAPuZcYWD8QgVHhQEA/IZXEoYzZ86cv1mA4+1++eUX5eXlyWKxaODAgQ7HmjRpIun8gGkAFVtevk3bj59x2McKzxWcS4XBwixJAGBSXkkYoqLO90NOSkpy2F+wZkHbtm1dFnULDg6WJBYlAyqBvUnnlJ1nc9jXiQpDxVbUom0WuqABgBl5JWFo3bq1JOnbb7912P/f//5XFotFffr0cTmnILlgPQag4nNesK1JXKSqRob4KBp4hcuibYxfAACz8srX90OGDNEvv/yiWbNmqU2bNurdu7fmzZunXbt2yWKx6Prrr3c5p2DsQr169bwRIgAfKmrAMyo4lwoD4xcAwKy8kjBMmDBBM2fOVGJioiZMmOBw7PLLL1e/fv1czlmyZIksFou6devmjRAB+JDLgGfWX6j4qDAAgN/wSpek2NhY/fDDD+rcubN9cTbDMNS7d28tWLDApf22bdu0YcMGSdKVV17pjRAB+MjpjFwdTMlw2BffgAHPFR4VBgDwG14bUdymTRtt3LhRhw4dUlJSkurUqaPGjRu7bT937lxJ59dnAFBxbT12xuF1aFCAWteJ9k0w8B4qDADgN7w+BVGTJk3sU6a607FjR3Xs2NFLEQHwJefuSB3qxSo40CvFT/gSFQYA8Bv8VgbgU84Dnhm/UEk4VxhYgwEATMvrFQabzaYVK1Zo3bp1SkpKUmZmpp5++mnVqVPH3iY3N1dWq1WBgYEKDeWXCFBR2WyGtjpNqcqCbZWEc4UhmAoDAJiVVxOGr776Svfff7+OHDnisP/hhx92SBjeffddTZw4UVFRUUpISFBkZKQ3wwTgJYdSM5SWbXXYx5SqlYRLhYExDABgVl7rkjR79mwNGzZMhw8flmEYql69ugzDKLLtnXfeqdjYWKWnp+uLL77wVogAvGyL0/iFWjGhqhPLB8dKwZrj+JpBzwBgWl5JGH7//Xfdd999ks7PerRr1y4lJye7bR8SEqIbbrhBhmFo2bJl3ggRgA84r/Ac36CqLBaLj6KBV+U5VxjokgQAZuWVhOHll1+W1WpVu3bt9PXXX6t169YlntO7d29J0pYtWzwdHgAfca4wdGLAc+VhdR7DQIUBAMzKKwnDjz/+KIvFogcffFAhISGlOqd58+aSpKNHj3oyNAA+kplr1d4T5xz2xTN+ofKgwgAAfsMrCcOxY8ckqUxrKxQMdM7MzPRITAB8a8exs8q3/TmOKTDAog71Y30YEbyKCgMA+A2vJAwFfZLL8uE/NTVVkhQbywcIoCJyXn+hde1oRYR4faZn+AoVBgDwG1757VyvXj39/vvvOnjwoH1sQknWrFkjSWratKknQwNQBMMwtDvxnNKy8zx2j5/2nXR4zXSqlQwVBgDwG15JGPr27at9+/bp/fff1+jRo0tsf/bsWb399tuyWCzq37+/FyIEUCA7L183zfpF25wqAJ7Ggm2VDBUGAPAbXumSdPfdd8tisWjVqlWaN29esW1TU1M1fPhwJSUlKSgoSPfcc483QgTwPx+uO+L1ZEGS4pkhqXKhwgAAfsMrCUN8fLweeOABGYah8ePH66abbtKCBQvsx3/++Wd9/PHHuu+++9S8eXP99NNPslgs+uc//6lGjRp5I0QA//Pfzce8fs8G1cLVpDorulcqeU4JAxUGADAtr40wnDFjhnJycvTWW2/ps88+02effWYfDH333Xfb2xWs/vzggw/q8ccf91Z4ACTtSkjTniTHqU5jw4MV4MG11BpWi9AT17VTgCdvAvOxOnVJosIAAKbltYTBYrHozTff1PDhw/Xcc89p1apVstlsLm0uv/xyPf7447rmmmu8FRqA//ncqbpQNzZMax7tz4d5lD8qDADgN7w+h+GVV16pK6+8UufOndOWLVuUnJys/Px8Va9eXZ06dVJcXJy3QwIgyZpv0+JtCQ77hsfXI1mAZzhXGIJCfRMHAKBEPpv0PDo6WldccYWvbg/AyZr9KTp5Lsdh3/Wd6/koGlRo+VbJZnXcF0yFAQDMyiuDngGY3+ebjzu87lg/Vs1rRvsoGlRoztUFSQpiDAMAmJVpEobTp0/r5MmT9kHPALznXHaevvstyWHf9Z3r+ygaVHjWHNd9VBgAwLQ8mjBYrVbt3LlTmzZt0smTJ12OZ2dna9q0aapfv77i4uJUu3ZtRUdH6y9/+Yt+++03T4YGoJBvdiQpx/rnJARBARYN7VjXhxGhQnNetE2iwgAAJuaRhMEwDE2bNk1xcXHq2LGjunfvrtq1a6tXr17asGGDJCk3N1dXXXWVnn76aSUmJsowDBmGoczMTH3xxRfq3r27li9f7onwADj5fIvj7Eh9W9VUtcgQH0WDCs950TaJCgMAmJhHBj2PHTtWH374oSQ5dDH6+eefdfXVV+vXX3/VzJkztXr1aklStWrV1KJFC1mtVu3atUtZWVnKysrSbbfdpr179yo2NtYTYQKQdOx0pn45eMph3w0MdoYnOVcYLIFSYLBvYgEAlKjcKwwrVqzQBx98IEkKDQ3VDTfcoIcfflgjR45UeHi4zpw5o5dfflnz5s1TcHCwZs2apZMnT2rdunXasGGDUlJS9PDDD0uSTp48qXnz5pV3iAAKWbTFcbBzTFiQ+rep6aNoUCk4VxioLgCAqZV7hWHu3LmSpJo1a+rHH39UmzZt7Mf27Nmj/v37a9asWbLZbHrkkUd05513OpwfHh6uF154QTt27NB3332npUuX6oEHHijvMAHofAXQeXakoR3rKjQo0EcRoVJwrjAwfgEATK3cKwy//vqrLBaL/v73vzskC5LUunVr/f3vf1d+fr4k6Y477nB7ndGjR0sSg58BD9p69IwOpmQ47GPtBXgcFQYA8CvlnjAkJJxfKfbyyy8v8njh/c2bN3d7nRYtWkiSTp065bYNgIvzhVN3pEbVI9S5YVUfRYNKgwoDAPiVck8YMjLOf1tZrVq1Io9XqVLFvh0aGur2OmFh53+B5Obmll9wAOxyrTZ9uS3BYd/18fVlsVh8FBEqDZcKAwkDAJiZx9ZhcPehgw8jgDms2JusM5l5DvtGxNMdCV7gUmGgSxIAmJlpVnoG4F2fb3Zce6F742pqWD3CR9GgUqHCAAB+hYQBqIROZ+Tqxz3JDvtGMNgZ3sIYBgDwKx5ZuE2SZs6cqZo1XedyT07+80PKU0895fb8wu0AlK+vdiQqL//PRRVDggI0uEMdH0aESsW5wkDCAACm5rGE4a233nJ7rGAcw5NPPump2wMohnN3pCvb1lJsOCvtwkucKwxMqwoApuaRhMEwjJIbAfCJgyfTteWPMw77bqA7ErzJmuP4mgoDAJhauScMK1asKO9LAihHzmsvVI8MUe8WNXwUDSolKxUGAPAn5Z4w9OnTp7wvCaCc2GyGPt/smDBc16muggOZ/wBelMcYBgDwJ3xKACqRDYdP6fgZx293b+hc30fRoNKiwgAAfoWEAahEnKsLLWtFqV3dGB9Fg0qLCgMA+BUSBqCSyM7L19IdiQ77ru9cn9XX4X0uC7dRYQAAMyNhACqJZbtOKD3Han9tsUjDOtX1YUSotFi4DQD8CgkDUEk4r73Qs1mc6sTyzS58gAoDAPgVEgagEkg+l63Vv6c47LuetRfgK1QYAMCvkDAAlcCXWxOUb/tzQcWIkEBd1a62DyNCpUaFAQD8CgkDUAk4z450dbvaigz1yELvQMmoMACAXyFhACq43Ylp2pWY5rDvetZegC9RYQAAv0LCAFRwX2xxrC7UjgnT5c2q+ygaVHqGUUSFIdQ3sQAASoWEAajAbDZDi7c6JgzD4+spMIC1F+Aj+bmSDMd9QVQYAMDMSBiACuzIqUydSMtx2MfsSPAp5+qCJAUzhgEAzIyEAajA9iY5jl2oHhmilrWifRQNIMma47qPCgMAmBoJA1CB7U1Kd3jdqjbJAnzMSoUBAPwNCQNQge094VhhIGGAz+Vlu+6jwgAApkbCAFRge5LOObxuTcIAX3OuMASGSAH8KgIAM+P/0kAFlZ2Xr8MpGQ77GL8An3OuMFBdAADTI2EAKqj9yemyOc1eScIAn3OuMDB+AQBMj4QBqKD2OnVHalgtQpGhQT6KBvgflwoDCQMAmB0JA1BB7T3hmDAw4Bmm4FJhoEsSAJgdCQNQQTHgGaZEhQEA/A4JA1BB7XNKGBi/AFOgwgAAfoeEAaiAzmbmKSnN8ZtcKgwwBSoMAOB3SBiACmhPkuOCbSGBAWocF+mjaIBCnCsMJAwAYHokDEAF5DzguVnNKAUH8uMOE3CuMDCtKgCYHp8ggArIecBzq1pRPooEcGJl4TYA8DckDEAF5DzguVXtGB9FAjhxThioMACA6ZEwABWMYRguXZIY8AzTcBn0TIUBAMyOhMEDli5dqieeeEJDhgxRmzZtFBcXp+DgYFWtWlVdunTRpEmTtHfv3lJf78iRI5o0aZJat26tyMhIVatWTd26ddOLL76ozMxMD74T+KOEs9k6l2112MeibTANl2lVqTAAgNkF+TqAisZqteraa68t8tiZM2e0efNmbd68Wa+//rqeeuopPfbYY8Veb8mSJbr99tuVlvbnrDeZmZnauHGjNm7cqHfffVdLly5V8+bNy/V9wH/tdZohKTosSHVi+VAGk6DCAAB+h4TBA2JjY9W3b19deumlatq0qerUqaOIiAglJCRo5cqVeu+993T27FlNmTJFVapU0T333FPkdbZs2aKbbrpJWVlZioqK0pQpU9SvXz9lZWVp/vz5mj17tvbt26chQ4Zo48aNio7mW2RIe5PSHV63qhUti8Xio2gAJ1QYAMDvkDCUs6CgIKWmpiowMLDI49ddd50mTpyoLl266PTp05o2bZruuuuuIts/8MADysrKUlBQkJYtW6bLL7/cfqx///5q0aKFJk+erH379mnGjBl64oknPPW24EecKwx0R4KpsHAbAPgdxjB4gLtkoUCTJk104403SpJOnjypPXv2uLRZv369Vq9eLUkaP368Q7JQYNKkSWrTpo0k6dVXX1VeXt7Fho4KwHlKVQY8w1RcKgx0SQIAsyNh8JHC3Yeys7Ndji9atMi+PXbs2CKvERAQoFGjRkk6Pz5ixYoV5Rsk/E5evk0HTjp1SWJKVZgJFQYA8DskDD6QlZWlxYsXSzr/ob9ly5YubdasWSNJioyMVJcuXdxeq0+fPvbttWvXlnOk8DeHUzKUl2847GtViwoDTIQKAwD4HRIGL8nLy9Mff/yh+fPnq0ePHvr9998lSePGjStysPLu3bslSc2bN1dQkPuhJq1bt3Y5B5WXc3ek2jFhio0I9lE0QBGoMACA32HQswcdPnxYTZo0cXv8qquu0owZM1z2Z2dnKyUlRZJUv379Yu9RtWpVRUZGKiMjQ0ePHi1TfMeOHSv2eGJiYpmuB9/b67LCM9UFmAwVBgDwOyQMPhAXF6c333xTN9xwQ5EDpM+d+/NDX1RUVInXK0gY0tPTS2xbWIMGDcrUHubHgGeYHhUGAPA7JAweVK9ePe3YsUPS+QXdjh8/rm+//VZz5szRPffcowMHDmjKlCku5xUeBB0SElLifUJDQyWdHxuBym3vCccpVVsyfgFmYrNJ+TmO+0gYAMD0Km3CUB4LWc2dO1djxoxxezw4OFjt27e3v+7UqZOGDBmiu+66S/369dPUqVP1+++/67333nM4Lyzsz1+gubm5JcaRk3P+F3B4eNlK+yV1YUpMTFT37t3LdE34TkaOVUdPOSaNdEmCqVhdZ4Rj4TYAML9KmzD40iWXXKL/+7//09/+9jfNnTtXN998swYNGmQ/XngQdGm6GWVkZEgqXfelwkoaHwH/su+EY3ekwACLmtcs238TgEcVlTAEMYYBAMyu0iYM5TGjUJ06dS743GHDhulvf/ubJOmzzz5zSBjCwsJUvXp1paamljgw+fTp0/aEgTEJlZvzgOfG1SMUFlz8IoKAV1FhAAC/VGkThsLTkfpCjRo17NtHjhxxOd62bVutXr1a+/fvl9VqdTu1auFVogtWfUbl5Dzgme5IMJ28IsZZUWEAANNjHQYfOX78uH27qK5EvXr1knS+u9GmTZvcXmfVqlX27Z49e5ZjhPA3zl2SWtVihWeYjEuFwSIFhfokFABA6ZEw+MjChQvt2x06dHA5Pnz4cPv23Llzi7yGzWbTBx98IEmqUqWK+vXrV75Bwq+wBgNMr6gpVcthAgoAgGeRMJSzRYsWlbjg2U8//aSnnnpKkhQUFKRbbrnFpU337t3Vu3dvSdKcOXO0bt06lzYzZsywj8V44IEHFBzMir6V1clzOUrNcJxRizUYYDoui7YxfgEA/EGlHcPgKYsWLdJNN92kIUOGaMCAAWrXrp2qVKminJwcHThwQEuWLNGCBQtks9kkSdOmTVOrVq2KvNarr76qnj17KisrS4MGDdLUqVPVr18/ZWVlaf78+Zo1a5YkqWXLlpo0aZLX3iPMx7m6EBYcoIbVInwUDeCGS4WB8QsA4A9IGDwgNzdXX3zxhb744gu3bcLDw/V///d/euihh9y2iY+P16effqrbb79daWlpmjp1qkubli1baunSpQ5TsaLy2ZPkumBbQABdPWAyVBgAwC+RMJSzF154QX369NFPP/2knTt36sSJE0pOTlZAQICqVaumdu3aqX///ho1alSppmUdOnSotm/frldffVVLly7VsWPHFBISoubNm2vkyJGaMGGCIiL4Jrmycx3wTAIJE6LCAAB+iYShnNWsWVNjx47V2LFjy+2ajRo10ksvvaSXXnqp3K6JioUBz/ALVBgAwC8x6BnwczaboX0nHFcEb12bKVVhQlQYAMAvkTAAfu6PU5nKyst32NeytuvaHoDPUWEAAL9EwgD4ub1O4xeqRYaoRhSLYcGEilqHAQBgeiQMgJ9zGb9QK1oWFsOCGTlXGEgYAMAvkDAAfo4Bz/AbzhUGuiQBgF8gYQD8nPMaDKzwDNOyMugZAPwRCQPgx7Lz8nU4NdNhX0sSBpiVc8JAhQEA/AIJA+DHDpxMV77NcNjXkkXbYFZ5zmMYqDAAgD8gYQD8mPP4hQbVwhUVynqMMCkqDADgl0gYAD/mOkMSC7bBxKgwAIBfImEA/NgelxmSWLANJkaFAQD8EgkD4Mf2nXBOGKgwwMSoMACAXyJhAPzU2cw8JZ51/MaWKVVhalQYAMAvMToS8FN7naoLwYEWNYmL9FE0QCk4L9xGhQHwmaysLKWlpSkjI0P5+fm+DgfFCAwMVGRkpGJiYhQe7pv/b5IwAH5qr9OCbc1qRCk4kKIhTMzq1CWJCgPgE2fPnlVCQoKvw0ApWa1W5eTk6NSpU6pbt65iY2O9HgMJA+CnnCsMreiOBLOjwgD4XFZWlkuyEBTEx0Ezs1qt9u2EhASFhoYqLMy7X7jwXwjgp1ymVCVhgNlRYQB8Li3tz+p0TEyMateurcDAQB9GhJLk5+crKSnJ/uzOnj3r9YSB/guAHzIMw2VKVQY8w9TyrZLN6rgviIQB8LaMjAz7NsmCfwgMDFTt2rXtrws/Q28hYQD8UOLZbJ3LdvzwxZSqMDXn6oJEwgD4QMEA56CgIJIFPxIYGGjvOuaLQeokDIAfch6/EB0apLqxfPiCiTmPX5CkYMYwAIA/IGEA/JDz+IWWtaNlsVh8FA1QCs5rMEhUGADAT5AwAH6IAc/wO0UlDFQYAMAvkDAAfogBz/A7eU5jGCyBUmCwb2IBAJQJCQPgZ/LybTqQnO6wr2UtEgaYnHOFgeoCABMYN26cLBaLqlevrpycnGLbbt26Vffcc4/atm2rmJgYhYSEqHbt2rryyis1Y8YMnTx50uUci8Xi8CcoKEh16tTR8OHD9dNPP3nqbZU71mEA/MyR1Azl5tsc9lFhgOk5VxgYvwDAx86dO6cFCxbIYrHo1KlTWrRokW666SaXdjabTZMnT9aMGTMUGBioK664QoMGDVJkZKSSk5O1bt06Pfzww5o+fbr27t2revXqOZxfvXp1TZgwQZKUnZ2trVu3avHixfryyy/16aefauTIkV55vxeDhAHwM87dkWrFhKpKRIiPogFKiQoDAJP59NNPlZGRoYceekivvPKK5syZU2TC8I9//EMzZsxQ586d9emnn6p58+YubTZv3qxHH31UWVmuU0jHxcXpiSeecNj37rvv6q677tLkyZP9ImGgSxLgZ1wHPLP+AvwAFQYAJjNnzhwFBQVp8uTJ6tevn5YvX64jR444tNm3b59efPFF1ahRQ99++22RyYIkde7cWd9//70aN25cqnuPGzdOkZGROnz4cJFdmcyGCgPgZ5wrDK1qRfkoEqAMXCoMJAyAmdhshk5n5vo6jFKrGhGigIALn058165d+uWXXzR48GDVqlVLo0aN0vLlyzV37lyHasD777+v/Px83X333apRo0aJ1y1YXK0s/GFadBIGwM/sO0GFAX7IpcJAlyTATE5n5qrL//3g6zBKbdPjA1U9KvSCz58zZ44k6Y477pAkXX/99frb3/6muXPnatq0aQoION8JZ926dZKkfv36XWTEjt5//31lZGSoSZMmiouLK9drewIJA+BHMnOt+uNUpsM+BjzDL1BhAGASeXl5+vDDDxUTE6Phw4dLkqKiojRixAh99NFH+uGHHzRo0CBJUlJSkiSpbt26LtdZuXKlVq5c6bCvb9++6tu3r8O+lJQUe9UiOztb27Zt07fffquAgAC9+OKL5frePIWEAfAj+06kyzD+fB1gkZrXpEsS/AAVBgAmsXjxYp08eVLjx49XWNifX16MGjVKH330kebMmWNPGIqzcuVKPfnkky77nROG1NRUe7vAwEDFxcVp2LBhmjRpknr37n1xb8ZLGPQM+JG9SWkOrxvHRSosONBH0QBlQIUBgEkUdEcaNWqUw/4BAwaoXr16Wrx4sU6dOiVJqlWrliQpISHB5TpPPPGEDMOQYRj65JNP3N6vVatW9nZWq1VJSUlatGiR3yQLEhUGwK+4DnimOxL8BLMkAaZWNSJEmx4f6OswSq3qBU4nfvToUS1btkyS1KdPH7ftPvroI91///3q0aOHVq5cqRUrVqh///4XdM+KgIQB8BPJadn6fPNxh32tGL8Af+FcYSBhAEwlIMByUYOI/cW8efNks9nUq1cvtWrVyuW41WrV+++/rzlz5uj+++/X6NGj9dxzz2nWrFl64IEH/GKAsieQMAB+wDAMTf1ip85m5Tns792i5CneAFNg4TYAPmYYhubOnSuLxaL3339fTZs2LbLdvn37tG7dOm3cuFFdu3bV5MmT9dxzz+maa67RJ598UuRaDGfOnPFw9L5FwgD4gS+3JeiH3Scc9l3Xsa66NKrqo4iAMsqjwgDAt3788UcdOnRIffr0cZssSNLYsWO1bt06zZkzR127dtXTTz+t3NxcvfTSS2rdurWuuOIKdezYUREREUpOTtb27du1fv16RUVFqVOnTt57Q17EoGfA5JLPZWv6l7857IuLCtET17XzUUTABbA6jWGgwgDAywoGO48ZM6bYdjfddJPCw8P1ySefKCsrSwEBAZoxY4Y2b96s8ePHKzExUe+++65efPFFLVmyRFFRUXrxxRd14MAB+zStFQ0VBsDEDMPQ41/s1JlMx65I/ze8vapFXtiAL8AnqDAA8LGPP/5YH3/8cYntYmJilJmZ6bI/Pj5e77zzTpnuaRSeC92PUWEATOzLbQlatsuxK9K1l9TR1e3r+Cgi4AIxhgEA/BYJA2BSJ8/luHRFqh4ZoifpigR/xLSqAOC3SBgAEzIMQ48v2uHSFelfw9tXimnvUAFRYQAAv0XCAJjQV9sT9d1vjl2RhlxSR4M70BUJfooKAwD4LRIGwGRS0nM0bfFOh33VIkP0FF2R4M+oMACA3yJhAExm2uKdOu3cFWkYXZHg56gwAIDfImEATGTp9kR9vSPJYd/gDrU15BK6IsHPUWEAAL9FwgCYREp6jv7p1BWpakSwnhrW3kcRAeXEMKgwAIAfI2EATGL64t90KiPXYd9Tw9orjq5I8Hf5uZKcFi8iYQAAv0HCAJjA0u2JWroj0WHf1e1q61q6IqEicK4uSFIwCQMA+AsSBsDHUouYFalqRLD+Nby9LBaLj6ICypHz+AVJCmIMAwD4CxIGwMemf/mbUp26Ij05rL1qRNMVCRVEUQkDFQYA8BskDIAPfbMjUV9td+yKdFW7WhpKVyRUJHlUGADAnwX5OgCgsrHm2/TT7yf12aZj+mFXssOxKnRFQkVkdRrDEBgiBfB9FQD4C/6PDXjJvhPn9MzXu3XZsz9q3LyN+npHknLzbQ5tnryunWpG01UDFYxzhYHqAgAfOnz4sCwWiywWi2rXri2r1Vpku927d9vbNW7c2L5/3rx59v3u/owZM8bhWhkZGXrmmWfUuXNnRUVFKTQ0VPXr11fv3r01ZcoUHThwwIPv+OJRYQA86Exmrr7clqDPNh3T9mNni207qG0tXdexrpciA7zIucLA+AUAJhAUFKQTJ07o66+/1nXXXedyfM6cOQoopho6YMAA9erVq8hjnTp1sm+fO3dOvXr10vbt29W8eXPdfvvtql69ulJSUrR+/Xo999xzatasmZo1a3bR78lTSBiAcubc5ci5iuAsJChAI7vU1z+vbUtXJFRMLhUGEgYAvtejRw9t27ZN7733nkvCYLVa9dFHH2ngwIFatWpVkecPHDhQjz32WIn3eeWVV7R9+3bdeeedmjVrlsvv+kOHDiknJ+fC34gXkDAARTAMQ19uS9D6Q6dkM4yST/ifXKuhn34/qZPnSv7B79Sgiv7Spb6GXlJXsRHBFxMuYG4uFQa6JAHwvfDwcN18882aM2eOkpOTVbNmTfuxr776SidOnNC4cePcJgyltW7dOknSfffdV+QXg02aNLmo63sDCQPgJC/fpqmf79DCTcfK/do1o0N1fef6+kuXempeM7rcrw+YEhUGwPxsNinrlK+jKL3wauUyecK4ceP0zjvv6MMPP9SkSZPs+9977z1Vq1ZNw4cPv+h7VK9eXZK0b98+h65K/oSEASgkI8eq+z7erJV7T5bbNUMCA3Rlu1r6S5f66t08TkGBzDWASoYKA2B+WaekF83bh97FIwekyLiLvkz37t3Vvn17zZ07154wJCUl6ZtvvtG9996r0FD3ayL98MMPys4uYtpoSTfffLNat24tSRo5cqQ++ugj3XnnnVq/fr0GDRqkLl262BMJf0DCAPxPSnqOxs3bUOLg5NLqaO9yVEdVIkLK5ZqAX6LCAMDExo0bp4ceeki//vqrLr30Ur3//vuyWq0aN25csectX75cy5cvL/JYp06d7AnDddddpxkzZmj69OmaMWOGZsyYIUlq1qyZrr76aj3wwANq0aJF+b6pckbCAEg6kpqhUe+t15HUTIf9YcEB+kuX+goqQ9kzLipEV7WrrRa16HIESKLCAMDUbr/9dj366KN67733dOmll2ru3LmKj48vsfvQs88+W6pBz5L00EMP6a677tK3336rn3/+WRs3btSvv/6qN998U3PmzNGnn35a5ExNZkHCgEpv+7EzGjt3g1Izch32V40I1ruju6lLo6o+igyoIKgwADCxGjVqaOjQoZo/f75GjhypvXv36vXXXy/3+0RHR2vkyJEaOXKkJOns2bOaOnWqZs6cqfHjx+v48eMKCTFnjwQSBlRqK/Ym677/bFZmbr7D/vpVw/X+uO5qViPKR5EBFYiVhAEwvfBq58cF+IvwauV6ufHjx+vzzz/XmDFjFBYWpttuu61cr1+U2NhYvfHGG1q6dKmOHDmiHTt2qEuXLh6/74UgYUCltXDjUT32+Q7l2xynTW1bJ0bzxnZTzRg+1ADlIo+F2wDTCwgol0HE/uqqq65SvXr1dPz4cd18882qWtU7vQssFosiIyO9cq+LQcKASscwDL25Yr/+vWyfy7FezeP01u2dFR3GughAubE6rUsSxBgGAOYSGBioRYsW6dixY+U+9ek777yjzp07q1u3bi7HFi1apN27d6tKlSpq3759ud63PJEwoFLJtxma/uVOffTLHy7Hhneqqxf+0lEhQUx7CpQrl0HPVBgAmE/Xrl3VtWvXUrcvblrV2rVr65577pEkffPNN7rnnnvUvHlz9ezZU3Xr1lVGRoa2bNmi1atXKyAgQDNnzix2CldfI2FApZGdl6/7P9miZbtOuBy7u09TPXpVawUEuK7ACOAiuQx6psIAwP8VN61qx44d7QnD888/r549e+r777/XTz/9pMTERElSvXr1NHr0aE2cONG0YxcKkDDANI7+vk0ph3Z45NqGIX2/K0k6ma5BhQsIFummrg00oLEh7fWjwV6APzl71PE1FQYAPtS4cWMZhlFyw/9xriKMGTNGY8aMKfX5rVq10iOPPKJHHnmk1OeYDQkDTOPYz5/q8kNveuz6nSWpqNnKtv3vDwDvYJYkAPArdNYGAHhXcISvIwAAlAEJAwDAu+qZu68uAMARXZJgGgHhVXXMUsej9wgJClC1yBAFMbgZ8L6wKlL3u6SarX0dCQCgDEgYYBqX3viIJP8dEAQAAFAR0SUJAAAAgFskDAAAAADcImEAAAAA4BYJAwAAQCURGBgoSbJarcrPz/dxNCit/Px8Wa1WSX8+Q28iYQAAAKgkIiMj7dtJSUkkDX4gPz9fSUlJ9teFn6G3MEsSAABAJRETE6NTp05JktLS0pSWlqagID4OmllBZaFAbGys12PgvxAAAIBKIjw8XHXr1lVCQoJ9n/MHUphX3bp1FRYW5vX7kjAAAABUIrGxsQoNDdXZs2eVkZFBtySTCwwMVGRkpGJjY32SLEgkDAAAAJVOWFiYzz58wv8w6BkAAACAWyQMAAAAANwiYQAAAADgFgkDAAAAALcY9Ay3Ck+zlpiY6MNIAAAAUJLCn9fKc7pcEga4dfLkSft29+7dfRgJAAAAyuLkyZNq3LhxuVyLLkkAAAAA3LIYhmH4OgiYU3Z2tnbs2CFJqlGjhumXjk9MTLRXQtavX686der4OCJ4C8++cuK5V148+8qJ514yq9Vq7yHSoUOHcltrw9yfAOFTYWFh6tatm6/DuCB16tRR/fr1fR0GfIBnXznx3Csvnn3lxHN3r7y6IRVGlyQAAAAAbpEwAAAAAHCLhAEAAACAWyQMAAAAANwiYQAAAADgFgkDAAAAALdIGAAAAAC4xcJtAAAAANyiwgAAAADALRIGAAAAAG6RMAAAAABwi4QBAAAAgFskDAAAAADcImEAAAAA4BYJAwAAAAC3SBgAAAAAuEXCAAAAAMAtEgYAAAAAbpEwwLSSk5P11Vdfadq0abrmmmsUFxcni8Uii8WiMWPGlPl633zzjUaMGKH69esrNDRU9evX14gRI/TNN9+Uf/C4YBs3btRTTz2lQYMG2Z9VVFSUWrZsqbFjx2rNmjVluh7P3T+kpaVp/vz5mjRpkvr06aPmzZsrNjZWISEhqlmzpvr27asXXnhBqamppbrezz//rNtvv12NGjVSWFiYateurauuukqffPKJh98JytOjjz5q//++xWLRypUrSzyHn3n/UfjZFvenb9++JV6L5+5hBmBSktz+GT16dKmvk5+fb4wfP77Y6915551Gfn6+594MSqV3797FPqeCP6NGjTJycnKKvRbP3b98//33pXr2cXFxxrffflvstaZPn24EBAS4vcaQIUOMrKwsL70zXKgtW7YYQUFBDs9uxYoVbtvzM+9/SvMzL8no06eP22vw3L2DhAGmVfiHvWHDhsagQYMuKGF47LHH7OfFx8cbn3zyibF+/Xrjk08+MeLj4+3HpkyZ4rk3g1Jp1qyZIcmoW7eu8cADDxifffaZsX79emPdunXGSy+9ZNSrV8/+vG655ZZir8Vz9y/ff/+90aBBA2PUqFHGq6++anz++efGunXrjLVr1xqffvqpMXLkSCMwMNCQZISEhBhbt24t8jpvv/22/dk2a9bMmDNnjrF+/Xpj0aJFRr9+/Ur93w98Kz8/3+jWrZshyahZs2apEgZ+5v1PwTO59957jR07drj9c/DgQbfX4Ll7BwkDTGvatGnGkiVLjKSkJMMwDOPQoUNlThj27t1r/4aqa9euRmZmpsPxjIwMo2vXroYkIygoyPj999/L+22gDIYMGWJ8+umnhtVqLfL4yZMnjZYtW9r/O1i1alWR7Xju/sfdMy/siy++sD/7ESNGuBxPTU01YmNj7V8ynDx50uUeQ4cOLdWHT/jWyy+/bEgyWrdubUyZMqXEZ8bPvH8qeK7Tp0+/oPN57t5DwgC/cSEJw7333ms/Z926dUW2Wbdunb3N3/72t3KMGJ6wZMkS+/OaOHFikW147hVXq1at7F2TnD3//PP2Z/rJJ58Uef7Ro0ftlYrBgwd7OlxcgCNHjhhRUVGGJGPlypXG9OnTS0wY+Jn3TxebMPDcvYdBz6iwDMPQ4sWLJUmtW7fWZZddVmS7yy67TK1atZIkLV68WIZheC1GlF2/fv3s2wcOHHA5znOv2KKjoyVJ2dnZLscWLVokSYqJidH1119f5Pn169fXwIEDJUnLly/XuXPnPBMoLth9992n9PR0jR49Wn369CmxPT/zlRPP3btIGFBhHTp0SAkJCZJU4i+dguPHjx/X4cOHPR0aLkJOTo59OzAw0OU4z73i2rt3r7Zu3Srp/AeEwnJzc7V+/XpJ0uWXX66QkBC31yl47jk5Odq4caNngsUFWbBggb766itVq1ZN//73v0t1Dj/zlRPP3btIGFBh7dq1y77t/OHCWeHju3fv9lhMuHirVq2yb7dp08blOM+9YsnMzNTvv/+ul156SX369JHVapUkPfjggw7t9u3bp/z8fEk8d3915swZPfDAA5Kk559/XnFxcaU6j595/7dw4UK1bdtWERERio6OVosWLTR69GitWLHC7Tk8d+8K8nUAgKccO3bMvl2/fv1i2zZo0MC+ffToUY/FhItjs9n03HPP2V/feOONLm147v5v3rx5Gjt2rNvjjz32mG699VaHfTx3/zd58mQlJSWpZ8+eGj9+fKnP49n7v8If/iVp//792r9/vz744AMNHz5c8+bNU2xsrEMbnrt3kTCgwircNzkqKqrYtpGRkfbt9PR0j8WEi/Pyyy/bu51cf/316tKli0sbnnvF1alTJ82aNUvdunVzOcZz92+rV6/Wu+++q6CgIL399tuyWCylPpdn778iIiJ03XXXacCAAWrdurWioqJ08uRJrVq1Sm+//bZSU1O1aNEiDRs2TN9//72Cg4Pt5/LcvYuEARVW4UGRxfVnlqTQ0FD7dlZWlsdiwoVbtWqVHnvsMUlSzZo19dZbbxXZjufu/4YPH66uXbtKOv9cDhw4oAULFuiLL77QLbfcoldeeUXXXnutwzk8d/+Vm5urv/71rzIMQ3//+9/Vvn37Mp3Ps/dfx48fV5UqVVz2X3nllZo4caKuueYabdmyRatWrdJbb72l+++/396G5+5djGFAhRUWFmbfzs3NLbZt4YG04eHhHosJF+a3337TiBEjZLVaFRYWpoULF6pmzZpFtuW5+78qVaqoffv2at++vbp166abb75Zn3/+uT744AMdPHhQw4YN07x58xzO4bn7r2eeeUZ79uxRw4YNNX369DKfz7P3X0UlCwVq1aqlzz77zF5VeP311x2O89y9i4QBFVbB9ItSySXIjIwM+3ZJpU1416FDhzRo0CCdPn1agYGBmj9/vq644gq37XnuFdcdd9yhkSNHymazacKECTp16pT9GM/dP+3Zs0fPPvuspPMfCAt3HSktnn3F1bRpU1155ZWSzo9rKJgVSeK5extdklBhFR4EVXhwVFEKD4IqPDgKvpWQkKCBAwcqISFBFotF7733noYNG1bsOTz3im3YsGFasGCBMjIy9O2339oHP/Pc/dPLL7+s3NxcNW3aVJmZmZo/f75Lm507d9q3f/zxRyUlJUmShg4dqsjISJ59Bde2bVt9/fXXks53Yapbt64kfua9jYQBFVbbtm3t23v27Cm2beHjRU3VCe9LSUnRlVdeqYMHD0o6/+3jqFGjSjyP516x1ahRw7595MgR+3bLli0VGBio/Px8nrsfKegqcvDgQd1yyy0ltv/Xv/5l3z506JAiIyP5ma/g3A2A57l7F12SUGE1adLE/k1E4bn7i/LTTz9JkurVq6fGjRt7OjSU4OzZs7rqqqvsU+0999xzuu+++0p1Ls+9Yjt+/Lh9u3DXgpCQEHXv3l2StG7dumL7NBf8dxEaGmofXA3/xc98xVZ4ytWC5yzx3L2NhAEVlsVisXdf2bNnj3755Zci2/3yyy/2bx+GDRtWpun8UP4yMzM1ZMgQbd68WZL0j3/8Q48++mipz+e5V2wLFy60b3fo0MHh2PDhwyVJaWlp+vzzz4s8/9ixY/rhhx8kSQMGDHDoBw3vmzdvngzDKPZP4YHQK1assO8v+ODHz3zFdejQIX3//feSpGbNmqlevXr2Yzx3LzMAP3Ho0CFDkiHJGD16dKnO2bt3rxEYGGhIMrp27WpkZmY6HM/MzDS6du1qSDKCgoKMffv2eSBylFZOTo4xaNAg+3N+4IEHLug6PHf/M3fuXCMrK6vYNi+99JL9v40mTZoYVqvV4XhqaqoRGxtrSDIaNWpkpKSkOBy3Wq3G0KFD7ddYsWJFeb8NeMD06dNLfGb8zPufL7/80sjLy3N7PCkpyYiPj7c/+xkzZri04bl7D2MYYFpr1qzR/v377a9TUlLs2/v373eZVnHMmDEu12jZsqUeeeQRPffcc9q4caN69uypRx99VM2aNdOBAwf0/PPPa8uWLZKkRx55RC1atPDIe0Hp3HLLLVq2bJkkqX///ho/frzDgEdnISEhatmypct+nrv/eeKJJzRp0iTdcMMN6tWrl5o1a6aoqCidO3dOO3bs0H/+8x+tXbtW0vnnPmvWLAUGBjpco1q1anr++ed1zz336MiRI7r00kv1j3/8Qx06dFBCQoJeeeUVrVixQtL5/9b69u3r7bcJD+Fn3v9MnDhReXl5uuGGG3T55ZercePGCg8PV0pKilauXKl33nnH/nu/V69eRXZL5bl7ka8zFsCd0aNH279ZKM0fd/Lz841x48YVe+748eON/Px8L747FKUsz1v/+xbZHZ67f2nUqFGpnnn9+vWNZcuWFXutadOmGRaLxe01Bg8eXGI1A+ZRmgqDYfAz729K+zN/ww03GKdPn3Z7HZ67d1gMwzAuKNMAPGzMmDF6//33S92+pP+Uv/76a82aNUsbNmxQSkqK4uLi1K1bN91999265pprLjZclIOy9i1t1KiRDh8+XGwbnrt/2Lt3r5YuXaq1a9dq//79OnHihFJTUxUeHq6aNWuqU6dOuvbaa3XjjTcqIiKixOv9/PPPevPNN7V69WqdOHFCVapUUceOHTV27NhSzcYD83jiiSf05JNPSjo/hqGkyhA/8/5h1apVWrVqldatW6eDBw8qJSVFaWlpioqKUoMGDdSjRw+NHj1al19+eamux3P3LBIGAAAAAG4xSxIAAAAAt0gYAAAAALhFwgAAAADALRIGAAAAAG6RMAAAAABwi4QBAAAAgFskDAAAAADcImEAAAAA4BYJAwAAAAC3SBgAAAAAuEXCAAAAAMAtEgYAAAAAbpEwAAAAAHCLhAEAAACAWyQMAAAAANwiYQAAAADgFgkDAKBCe+KJJ2SxWGSxWHwdCgD4JRIGAKiEDh8+bP8QfTF/AAAVHwkDAMCr5s2bZ084Dh8+7OtwKrSVK1fa/61Xrlzp63AA+KkgXwcAAPC+evXqaceOHW6Pd+jQQZLUtWtXzZ0711thAQBMiIQBACqh4OBgtW/fvsR2kZGRpWoHAKi46JIEAAAAwC0SBgBAmdlsNn300UcaPHiwateurZCQENWoUUP9+vXTzJkzlZub63JOQX/6sWPH2vc1adLEZSC1c1/7X375RY8//rj69u1rv1dMTIzatm2re++9V7t27fL027U7d+6cZsyYof79+zvEEh8fr4kTJ2rt2rVuzz158qQef/xxxcfHq0qVKgoLC1Pjxo11xx13aM2aNSXe+8cff9Qtt9yiJk2aKDw8XBEREWrUqJEuu+wyPfzww/rxxx/tbQsGtffr18++r1+/fi7/1vPmzbuofw8AlYQBAIATSYYko0+fPi7HUlNTjZ49e9rbFPWnTZs2xuHDhx3OW7FiRbHnFPxZsWKF/Zy5c+eW2D4wMNB488033b6X6dOn29tejO+//96Ii4srMZ6ifPfdd0ZMTEyx5913331Gfn5+kec/+OCDJd63evXq9vaHDh0q1b/13LlzL+rfBEDlwBgGAECp5efn69prr9W6deskSX369NGECRPUpEkTJSQk6L333tOiRYu0e/duDRgwQFu3blVUVJQkqVu3btqxY4cWL16sxx9/XJL03XffqW7dug73aNKkiX3barWqatWqGjZsmK644gq1aNFCkZGRSkhI0ObNm/Xaa68pJSVFEyZMUOvWrdW/f3+PvO8VK1bommuukdVqVWBgoO644w4NGzZMDRs2VHZ2tnbt2qVvvvlGS5YscTl369atGjp0qHJzcxUcHKwJEybouuuuU2RkpLZs2aLnnntOhw4d0ptvvqnIyEg9//zzDud/9dVXeuWVVyRJl1xyie699161adNGsbGxOnPmjH777Tf98MMPWr9+vf2cgkHtGzZs0Lhx4yRJ7733nrp16+Zw7fr165fzvxSACsnXGQsAwHzkpsLwxhtv2I+NGjXKsNlsLudOnTrV3mby5MkuxwtXDQ4dOlRsHMeOHTMyMjLcHj9z5oxxySWXGJKMXr16FdnmYisMWVlZRt26dQ1JRkREhEMFxNkff/zhsq9bt272Ssh3333ncvzUqVNG27ZtDUlGQECAsXPnTofjd9xxhyHJaNSokXHu3Dm3905NTXXZV7iqU1zcAFAcxjAAAErtzTfflCTVqFFDb7zxRpGLtz355JNq3bq1JGn27NnKycm54PvVq1dPERERbo/HxsbqqaeekiStWbNGqampF3wvdz744AMlJCRIkp555hn17dvXbdsGDRo4vF6/fr02bNggSbrrrrs0aNAgl3OqVq2qWbNmSTo/NmTmzJkOx5OSkiRJnTt3tldrilKtWrWS3wwAXAASBgBAqSQkJGj37t2SpBtvvFHR0dFFtgsKCrIPbD59+rQ2b95cbjFkZGTo8OHD+u2337Rz507t3LlTwcHB9uPbtm0rt3sV+OqrrySdn2L2rrvuKtO5P/zwg317/Pjxbtv17NlTbdq0cTlHkurUqSNJ+umnn3TgwIEy3R8AygMJAwCgVHbu3GnfvvTSS4ttW/h44fMuREpKiqZOnapWrVopOjpaTZo0Ufv27dWhQwd16NBBQ4YMcWhb3rZs2SJJ6tKlS7HVjqIUvPeQkBB16tSp2LYF/2a///67wyxTo0aNkiSlpqaqffv2uvnmmzV37lzt37+/TLEAwIUiYQAAlMqpU6fs2zVr1iy2be3atYs8r6w2bdqk1q1b69lnn9W+fftkGEax7bOysi74Xu4UJCEF3/SXRcF7r1atmoKCip9npODfzDAMnT592r5/wIABeuONNxQeHq7s7Gx9+umnGjdunFq0aKH69evrnnvu8UhlBQAKkDAAAMqsqLEL5S03N1c33nijUlNTFRwcrIceekirVq1SYmKisrOzZRiGDMNw6KZTUkLhKxf773Xffffp8OHDevnllzV48GDFxsZKko4fP6533nlH8fHx9pmnAKC8kTAAAEql8KDaEydOFNu2YKCu83ll8eOPP+rgwYOSpJkzZ2rGjBm64oorVLt2bYWGhtrbXUwFozTi4uIkSYmJiWU+t+C9p6amymq1Ftu24N/MYrGoatWqLsdr1qypBx98UEuXLtWpU6e0adMmPf7446pSpYoMw9DTTz+txYsXlzlGACgJCQMAoFTat29v3/7111+LbVt4TYDC50ml/7b9t99+s2/fdNNNbttt3LixVNe7UJ07d7bfJzMzs0znFrz33Nxcbd26tdi2Bf9mLVq0UEhISLFtAwIC1LlzZ/3rX//S8uXL7fsXLFjg0M4blSAAFR8JAwCgVOrWrWufyWfBggVKT08vsl1+fr7mzZsn6fyUoQUfuAuEhYXZt4ubcrXwN/IZGRlFtrHZbJo9e3ap4r9QQ4cOlSRlZmbapz8trYEDB9q333vvPbft1q1bp127drmcUxqdO3e2VyScB32X9t8aAIpDwgAAKLX77rtPknTy5Endf//9RbZ58skn7R9+77rrLofuQ5Lj4OHipglt0aKFfbsgAXE2ZcqUcp22tSi333676tWrJ0n6xz/+oVWrVrlte+zYMYfX3bt3V9euXSWdX5OicDWgwNmzZ3X33XdLOl85uPfeex2Of/rpp8UO5t64caN9kHThVbKl0v9bA0BxLIZZR4gBAHymoCtLnz59tHLlSvv+/Px89e7dW+vWrZMk9e/fX3/729/UpEkTJSYm6r333tPnn38uSWrWrJm2bt3qstjYuXPnVLNmTWVnZ6tz58567rnn1KhRIwUEnP8Oq169egoPD1dGRoaaNm2q5ORkBQYG6s4779SIESMUFxen/fv32z+A9+zZU2vXrpUkzZ07V2PGjHG43xNPPKEnn3xS0oUPil6xYoUGDRokq9WqoKAg3XHHHRo+fLjq16+vnJwc7dmzR19//bW+/PJLl2/yt27dqksvvVS5ubkKCQnRxIkTNXToUEVGRmrLli167rnn7GM1Jk+erOeff97h/MaNG+vs2bMaNmyYrrjiCrVs2VKRkZFKTU3VmjVr9Prrr+vUqVMKDAzUL7/8Yk9QCjRo0EDHjh1TkyZN9Morr6hVq1YKDAyUJNWqVcvtehoAYOezNaYBAKYlyZBk9OnTx+VYamqq0bNnT3ubov60adPGOHz4sNvrT5482e25K1assLf79ttvjbCwMLdt+/bta+zcudP+eu7cuS73mj59uv34xfj222+NqlWrFvu+3d3ju+++M2JiYoo977777jPy8/Ndzm3UqFGJ9wwNDS3yvRuGYcycOdPtee7OAYDC6JIEACiTatWq6aefftIHH3ygq6++WrVq1VJwcLCqV6+uvn376o033tDWrVvVqFEjt9d47rnnNHv2bPXu3VvVqlWzf+Pt7KqrrtLGjRt1++23q27dugoODlaNGjXUp08fzZo1S8uXL1dkZKSn3qpLLAcPHtQzzzyjHj16qHr16goMDFRMTIw6d+6sBx980GGwd2GDBg3S/v37NXXqVHXq1EkxMTEKDQ1Vw4YNddttt2n16tV644037FWWwlasWKFXX31VN9xwgzp06KAaNWooKChIMTExio+P18MPP6xdu3a5VFYK3Hvvvfrvf/+rQYMGqWbNmiWuBwEAzuiSBAAAAMAtKgwAAAAA3CJhAAAAAOAWCQMAAAAAt0gYAAAAALhFwgAAAADALRIGAAAAAG6RMAAAAABwi4QBAAAAgFskDAAAAADcImEAAAAA4BYJAwAAAAC3SBgAAAAAuEXCAAAAAMAtEgYAAAAAbpEwAAAAAHCLhAEAAACAWyQMAAAAANwiYQAAAADgFgkDAAAAALdIGAAAAAC4RcIAAAAAwC0SBgAAAABukTAAAAAAcIuEAQAAAIBbJAwAAAAA3Pp/q+TUfDaFnUIAAAAASUVORK5CYII=" }, "metadata": {}, "output_type": "display_data" @@ -680,8 +696,8 @@ "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2024-01-29T08:23:44.635831300Z", - "start_time": "2024-01-29T08:23:44.193377300Z" + "end_time": "2024-02-08T08:03:49.468051600Z", + "start_time": "2024-02-08T08:03:49.206250200Z" } }, "id": "e3604083ed32e0eb" From cc3ad51ffae2a24adde72337ff940dc9ab6e7281 Mon Sep 17 00:00:00 2001 From: Andrea Ponti <59694427+andreaponti5@users.noreply.github.com> Date: Thu, 8 Feb 2024 09:22:58 +0100 Subject: [PATCH 19/20] Update botorch_community/models/gp_regression_multisource.py Co-authored-by: Max Balandat --- botorch_community/models/gp_regression_multisource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/botorch_community/models/gp_regression_multisource.py b/botorch_community/models/gp_regression_multisource.py index 9870e24c5e..b838be3745 100644 --- a/botorch_community/models/gp_regression_multisource.py +++ b/botorch_community/models/gp_regression_multisource.py @@ -105,7 +105,7 @@ def __init__( train_Y: A `batch_shape x n x m` tensor of training observations. train_Yvar: A `batch_shape x n x m` tensor of observed measurement noise. - m: The moltiplicator factor of the model standard deviation used to select + m: The multiplication factor of the model standard deviation used to select points from other sources to add to the Augmented GP. likelihood: A likelihood. If omitted, use a standard GaussianLikelihood with inferred noise level. From 88a457fdc099e1d9a65c980ec3fc6c4afa1e1c1f Mon Sep 17 00:00:00 2001 From: Andrea Ponti Date: Sun, 11 Feb 2024 15:41:43 +0100 Subject: [PATCH 20/20] Change SingleTaskAugmentedGP description --- .../models/gp_regression_multisource.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/botorch_community/models/gp_regression_multisource.py b/botorch_community/models/gp_regression_multisource.py index b838be3745..376d1e9142 100644 --- a/botorch_community/models/gp_regression_multisource.py +++ b/botorch_community/models/gp_regression_multisource.py @@ -76,14 +76,15 @@ def get_random_x_for_agp( class SingleTaskAugmentedGP(SingleTaskGP): r"""A single-task multi-source GP model. - The Augmented Gaussian Process is described in [Ca2021ms]_. - The basic idea is to use GP sparsification for selecting a subset of - the function evaluations, among those performed so far over all the - different sources, as inducing locations to generate the AGP approximating f(x). - The GP sparsification proposed is an insertion method: the set of inducing - locations is initialized with the function evaluations on the most expensive - information source and is incremented by including evaluations on other sources - depending on both a model discrepancy measure and GP’s predictive uncertainty. + The Augmented Gaussian Process (AGP) is described in [Ca2021ms]_. + The basic idea is to select a subset of function evaluations (namely, + augmenting observations), among those performed so far over all the + cheap sources, to augment the set of observations from the ground + truth source. The resulting augmented set of observations is used to + fit the AGP. + The set of "augmenting observations" is selected depending on a + discrepancy measure between GP's predictive mean and GP’s predictive + uncertainty. """ def __init__(