From ac7de648ea0f7c9ba4163849e17a15f7a91bda91 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Wed, 4 Dec 2024 15:29:53 +0100 Subject: [PATCH 01/43] kernels working on a given set of features --- bofire/data_models/kernels/aggregation.py | 8 ++--- bofire/data_models/kernels/api.py | 11 ++++-- bofire/data_models/kernels/categorical.py | 6 ++-- bofire/data_models/kernels/continuous.py | 9 ++--- bofire/data_models/kernels/kernel.py | 10 ++++++ bofire/data_models/kernels/molecular.py | 6 ++-- bofire/kernels/mapper.py | 36 ++++++++++++++++++- bofire/surrogates/mixed_single_task_gp.py | 8 ++++- bofire/surrogates/mixed_tanimoto_gp.py | 16 +++++++-- bofire/surrogates/multi_task_gp.py | 3 ++ bofire/surrogates/shape.py | 9 +++++ bofire/surrogates/single_task_gp.py | 3 ++ tests/bofire/data_models/specs/kernels.py | 11 +++++- tests/bofire/kernels/test_mapper.py | 10 ++++++ tests/bofire/strategies/doe/test_design.py | 1 + tests/bofire/strategies/test_doe.py | 3 ++ .../bofire/strategies/test_from_data_model.py | 3 ++ tests/bofire/strategies/test_space_filling.py | 3 ++ tests/bofire/surrogates/test_xgb.py | 3 ++ 19 files changed, 138 insertions(+), 21 deletions(-) diff --git a/bofire/data_models/kernels/aggregation.py b/bofire/data_models/kernels/aggregation.py index 58afe9a8d..d7751ef3b 100644 --- a/bofire/data_models/kernels/aggregation.py +++ b/bofire/data_models/kernels/aggregation.py @@ -3,13 +3,13 @@ from bofire.data_models.kernels.categorical import HammingDistanceKernel from bofire.data_models.kernels.continuous import LinearKernel, MaternKernel, RBFKernel -from bofire.data_models.kernels.kernel import Kernel +from bofire.data_models.kernels.kernel import AggregationKernel, Kernel from bofire.data_models.kernels.molecular import TanimotoKernel from bofire.data_models.kernels.shape import WassersteinKernel from bofire.data_models.priors.api import AnyGeneralPrior -class AdditiveKernel(Kernel): +class AdditiveKernel(AggregationKernel): type: Literal["AdditiveKernel"] = "AdditiveKernel" kernels: Sequence[ Union[ @@ -26,7 +26,7 @@ class AdditiveKernel(Kernel): type: Literal["AdditiveKernel"] = "AdditiveKernel" -class MultiplicativeKernel(Kernel): +class MultiplicativeKernel(AggregationKernel): type: Literal["MultiplicativeKernel"] = "MultiplicativeKernel" kernels: Sequence[ Union[ @@ -42,7 +42,7 @@ class MultiplicativeKernel(Kernel): ] -class ScaleKernel(Kernel): +class ScaleKernel(AggregationKernel): type: Literal["ScaleKernel"] = "ScaleKernel" base_kernel: Union[ RBFKernel, diff --git a/bofire/data_models/kernels/api.py b/bofire/data_models/kernels/api.py index 609f76f50..ce37ad223 100644 --- a/bofire/data_models/kernels/api.py +++ b/bofire/data_models/kernels/api.py @@ -17,12 +17,19 @@ PolynomialKernel, RBFKernel, ) -from bofire.data_models.kernels.kernel import Kernel +from bofire.data_models.kernels.kernel import AggregationKernel, ConcreteKernel, Kernel from bofire.data_models.kernels.molecular import MolecularKernel, TanimotoKernel from bofire.data_models.kernels.shape import WassersteinKernel -AbstractKernel = Union[Kernel, CategoricalKernel, ContinuousKernel, MolecularKernel] +AbstractKernel = Union[ + Kernel, + CategoricalKernel, + ContinuousKernel, + MolecularKernel, + ConcreteKernel, + AggregationKernel, +] AnyContinuousKernel = Union[ MaternKernel, diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index 1bd39313d..e3ef4c5d9 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -1,9 +1,9 @@ -from typing import Literal +from typing import List, Literal, Optional -from bofire.data_models.kernels.kernel import Kernel +from bofire.data_models.kernels.kernel import ConcreteKernel -class CategoricalKernel(Kernel): +class CategoricalKernel(ConcreteKernel): pass diff --git a/bofire/data_models/kernels/continuous.py b/bofire/data_models/kernels/continuous.py index f081a239b..3516648e7 100644 --- a/bofire/data_models/kernels/continuous.py +++ b/bofire/data_models/kernels/continuous.py @@ -1,12 +1,12 @@ -from typing import Literal, Optional +from typing import List, Literal, Optional from pydantic import PositiveInt, field_validator -from bofire.data_models.kernels.kernel import Kernel +from bofire.data_models.kernels.kernel import ConcreteKernel from bofire.data_models.priors.api import AnyGeneralPrior, AnyPrior -class ContinuousKernel(Kernel): +class ContinuousKernel(ConcreteKernel): pass @@ -40,6 +40,7 @@ class PolynomialKernel(ContinuousKernel): power: int = 2 -class InfiniteWidthBNNKernel(Kernel): +class InfiniteWidthBNNKernel(ContinuousKernel): + features: Optional[List[str]] = None type: Literal["InfiniteWidthBNNKernel"] = "InfiniteWidthBNNKernel" depth: PositiveInt = 3 diff --git a/bofire/data_models/kernels/kernel.py b/bofire/data_models/kernels/kernel.py index 18918e562..5c74918cc 100644 --- a/bofire/data_models/kernels/kernel.py +++ b/bofire/data_models/kernels/kernel.py @@ -1,5 +1,15 @@ +from typing import List, Literal, Optional + from bofire.data_models.base import BaseModel class Kernel(BaseModel): type: str + + +class AggregationKernel(Kernel): + pass + + +class ConcreteKernel(Kernel): + features: Optional[List[str]] = None diff --git a/bofire/data_models/kernels/molecular.py b/bofire/data_models/kernels/molecular.py index 522986f2b..5e0a371a1 100644 --- a/bofire/data_models/kernels/molecular.py +++ b/bofire/data_models/kernels/molecular.py @@ -1,9 +1,9 @@ -from typing import Literal +from typing import List, Literal, Optional -from bofire.data_models.kernels.kernel import Kernel +from bofire.data_models.kernels.kernel import ConcreteKernel -class MolecularKernel(Kernel): +class MolecularKernel(ConcreteKernel): pass diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index fe1e62622..f05baf790 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Callable, List, Optional import gpytorch import torch @@ -11,12 +11,25 @@ from bofire.kernels.shape import WassersteinKernel +def _compute_active_dims( + data_model: data_models.ConcreteKernel, + active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], +) -> List[int]: + if data_model.features: + assert features_to_idx_mapper is not None + active_dims = features_to_idx_mapper(data_model.features) + return active_dims + + def map_RBFKernel( data_model: data_models.RBFKernel, batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> gpytorch.kernels.RBFKernel: + active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) return gpytorch.kernels.RBFKernel( batch_shape=batch_shape, ard_num_dims=len(active_dims) if data_model.ard else None, @@ -34,7 +47,9 @@ def map_MaternKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> gpytorch.kernels.MaternKernel: + active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) return gpytorch.kernels.MaternKernel( batch_shape=batch_shape, ard_num_dims=len(active_dims) if data_model.ard else None, @@ -53,6 +68,7 @@ def map_InfiniteWidthBNNKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> "InfiniteWidthBNNKernel": # type: ignore # noqa: F821 try: from botorch.models.kernels.infinite_width_bnn import ( # type: ignore @@ -66,6 +82,7 @@ def map_InfiniteWidthBNNKernel( "requires python 3.10+.", ) + active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) return InfiniteWidthBNNKernel( batch_shape=batch_shape, active_dims=tuple(active_dims), @@ -78,7 +95,9 @@ def map_LinearKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> gpytorch.kernels.LinearKernel: + active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) return gpytorch.kernels.LinearKernel( batch_shape=batch_shape, active_dims=active_dims, @@ -95,7 +114,9 @@ def map_PolynomialKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> gpytorch.kernels.PolynomialKernel: + active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) return gpytorch.kernels.PolynomialKernel( batch_shape=batch_shape, active_dims=active_dims, @@ -113,6 +134,7 @@ def map_AdditiveKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> gpytorch.kernels.AdditiveKernel: return gpytorch.kernels.AdditiveKernel( *[ # type: ignore @@ -121,6 +143,7 @@ def map_AdditiveKernel( batch_shape=batch_shape, ard_num_dims=ard_num_dims, active_dims=active_dims, + features_to_idx_mapper=features_to_idx_mapper, ) for k in data_model.kernels ], @@ -132,6 +155,7 @@ def map_MultiplicativeKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> gpytorch.kernels.ProductKernel: return gpytorch.kernels.ProductKernel( *[ # type: ignore @@ -140,6 +164,7 @@ def map_MultiplicativeKernel( batch_shape=batch_shape, ard_num_dims=ard_num_dims, active_dims=active_dims, + features_to_idx_mapper=features_to_idx_mapper, ) for k in data_model.kernels ], @@ -151,6 +176,7 @@ def map_ScaleKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> gpytorch.kernels.ScaleKernel: return gpytorch.kernels.ScaleKernel( base_kernel=map( @@ -158,6 +184,7 @@ def map_ScaleKernel( batch_shape=batch_shape, ard_num_dims=ard_num_dims, active_dims=active_dims, + features_to_idx_mapper=features_to_idx_mapper, ), outputscale_prior=( priors.map(data_model.outputscale_prior) @@ -172,7 +199,9 @@ def map_TanimotoKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> TanimotoKernel: + active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) return TanimotoKernel( batch_shape=batch_shape, ard_num_dims=len(active_dims) if data_model.ard else None, @@ -185,7 +214,9 @@ def map_HammingDistanceKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> CategoricalKernel: + active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) return CategoricalKernel( batch_shape=batch_shape, ard_num_dims=len(active_dims) if data_model.ard else None, @@ -198,6 +229,7 @@ def map_WassersteinKernel( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> WassersteinKernel: return WassersteinKernel( squared=data_model.squared, @@ -230,10 +262,12 @@ def map( batch_shape: torch.Size, ard_num_dims: int, active_dims: List[int], + features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> GpytorchKernel: return KERNEL_MAP[data_model.__class__]( data_model, batch_shape, ard_num_dims, active_dims, + features_to_idx_mapper, ) diff --git a/bofire/surrogates/mixed_single_task_gp.py b/bofire/surrogates/mixed_single_task_gp.py index 45e772637..79079f74f 100644 --- a/bofire/surrogates/mixed_single_task_gp.py +++ b/bofire/surrogates/mixed_single_task_gp.py @@ -92,7 +92,13 @@ def _fit(self, X: pd.DataFrame, Y: pd.DataFrame): train_Y=tY, cat_dims=cat_dims, # cont_kernel_factory=self.continuous_kernel.to_gpytorch, - cont_kernel_factory=partial(kernels.map, data_model=self.continuous_kernel), + cont_kernel_factory=partial( + kernels.map, + data_model=self.continuous_kernel, + features_to_idx_mapper=lambda feats: self.inputs.get_feature_indices( + self.input_preprocessing_specs, feats + ), + ), outcome_transform=( Standardize(m=tY.shape[-1]) if self.output_scaler == ScalerEnum.STANDARDIZE diff --git a/bofire/surrogates/mixed_tanimoto_gp.py b/bofire/surrogates/mixed_tanimoto_gp.py index 8abe5c2e9..45dc4b519 100644 --- a/bofire/surrogates/mixed_tanimoto_gp.py +++ b/bofire/surrogates/mixed_tanimoto_gp.py @@ -317,9 +317,21 @@ def _fit(self, X: pd.DataFrame, Y: pd.DataFrame): # type: ignore train_Y=tY, cat_dims=cat_dims, mol_dims=mol_dims, - cont_kernel_factory=partial(kernels.map, data_model=self.continuous_kernel), + cont_kernel_factory=partial( + kernels.map, + data_model=self.continuous_kernel, + features_to_idx_mapper=lambda feats: self.inputs.get_feature_indices( + self.input_preprocessing_specs, feats + ), + ), # cat_kernel_factory=partial(kernels.map, data_model=self.categorical_kernel), BoTorch forced to use CategoricalKernel - mol_kernel_factory=partial(kernels.map, data_model=self.molecular_kernel), + mol_kernel_factory=partial( + kernels.map, + data_model=self.molecular_kernel, + features_to_idx_mapper=lambda feats: self.inputs.get_feature_indices( + self.input_preprocessing_specs, feats + ), + ), outcome_transform=Standardize(m=tY.shape[-1]), input_transform=tf, ) diff --git a/bofire/surrogates/multi_task_gp.py b/bofire/surrogates/multi_task_gp.py index 26bc3dd0a..25f9e4e10 100644 --- a/bofire/surrogates/multi_task_gp.py +++ b/bofire/surrogates/multi_task_gp.py @@ -70,6 +70,9 @@ def _fit(self, X: pd.DataFrame, Y: pd.DataFrame): # type: ignore range(tX.shape[1] - 1), ), # kernel is for input space so we subtract one for the fidelity index ard_num_dims=1, # this keyword is ignored + features_to_idx_mapper=lambda feats: self.inputs.get_feature_indices( + self.input_preprocessing_specs, feats + ), ), outcome_transform=( Standardize(m=tY.shape[-1]) diff --git a/bofire/surrogates/shape.py b/bofire/surrogates/shape.py index b708a85a0..d86c25d46 100644 --- a/bofire/surrogates/shape.py +++ b/bofire/surrogates/shape.py @@ -96,12 +96,18 @@ def _fit(self, X: pd.DataFrame, Y: pd.DataFrame): active_dims=self.idx_continuous, ard_num_dims=1, batch_shape=torch.Size(), + features_to_idx_mapper=lambda feats: self.inputs.get_feature_indices( + self.input_preprocessing_specs, feats + ), ) * kernels.map( self.shape_kernel, active_dims=self.idx_shape, ard_num_dims=1, batch_shape=torch.Size(), + features_to_idx_mapper=lambda feats: self.inputs.get_feature_indices( + self.input_preprocessing_specs, feats + ), ), outputscale_prior=priors.map(self.outputscale_prior), ) @@ -112,6 +118,9 @@ def _fit(self, X: pd.DataFrame, Y: pd.DataFrame): active_dims=self.idx_shape, ard_num_dims=1, batch_shape=torch.Size(), + features_to_idx_mapper=lambda feats: self.inputs.get_feature_indices( + self.input_preprocessing_specs, feats + ), ), outputscale_prior=priors.map(self.outputscale_prior), ) diff --git a/bofire/surrogates/single_task_gp.py b/bofire/surrogates/single_task_gp.py index 8764ebbdd..ee710586a 100644 --- a/bofire/surrogates/single_task_gp.py +++ b/bofire/surrogates/single_task_gp.py @@ -53,6 +53,9 @@ def _fit(self, X: pd.DataFrame, Y: pd.DataFrame): batch_shape=torch.Size(), active_dims=list(range(tX.shape[1])), ard_num_dims=1, # this keyword is ignored + features_to_idx_mapper=lambda feats: self.inputs.get_feature_indices( + self.input_preprocessing_specs, feats + ), ), outcome_transform=( Standardize(m=tY.shape[-1]) diff --git a/tests/bofire/data_models/specs/kernels.py b/tests/bofire/data_models/specs/kernels.py index 2056673af..96475a811 100644 --- a/tests/bofire/data_models/specs/kernels.py +++ b/tests/bofire/data_models/specs/kernels.py @@ -10,6 +10,7 @@ kernels.HammingDistanceKernel, lambda: { "ard": True, + "features": None, }, ) specs.add_valid( @@ -21,13 +22,17 @@ ) specs.add_valid( kernels.LinearKernel, - lambda: {"variance_prior": priors.valid(GammaPrior).obj().model_dump()}, + lambda: { + "variance_prior": priors.valid(GammaPrior).obj().model_dump(), + "features": None, + }, ) specs.add_valid( kernels.MaternKernel, lambda: { "ard": True, "nu": 2.5, + "features": None, "lengthscale_prior": priors.valid().obj().model_dump(), }, ) @@ -37,6 +42,7 @@ "ard": True, "nu": 5, "lengthscale_prior": priors.valid().obj(), + "features": None, }, error=ValueError, message="nu expected to be 0.5, 1.5, or 2.5", @@ -45,6 +51,7 @@ kernels.InfiniteWidthBNNKernel, lambda: { "depth": 3, + "features": None, }, ) @@ -53,6 +60,7 @@ lambda: { "ard": True, "lengthscale_prior": priors.valid().obj().model_dump(), + "features": None, }, ) specs.add_valid( @@ -84,5 +92,6 @@ kernels.TanimotoKernel, lambda: { "ard": True, + "features": None, }, ) diff --git a/tests/bofire/kernels/test_mapper.py b/tests/bofire/kernels/test_mapper.py index 79cbd9bfd..fec7d8341 100644 --- a/tests/bofire/kernels/test_mapper.py +++ b/tests/bofire/kernels/test_mapper.py @@ -54,6 +54,7 @@ def test_map(kernel_spec: Spec): batch_shape=torch.Size(), ard_num_dims=10, active_dims=list(range(5)), + features_to_idx_mapper=None, ) assert isinstance(gkernel, EQUIVALENTS[kernel.__class__]) @@ -66,6 +67,7 @@ def test_map_infinite_width_bnn_kernel(): batch_shape=torch.Size(), active_dims=list(range(5)), ard_num_dims=10, + features_to_idx_mapper=None, ) assert isinstance(gkernel, BNNKernel) @@ -79,6 +81,7 @@ def test_map_scale_kernel(): batch_shape=torch.Size(), ard_num_dims=10, active_dims=list(range(5)), + features_to_idx_mapper=None, ) assert hasattr(k, "outputscale_prior") assert isinstance(k.outputscale_prior, gpytorch.priors.GammaPrior) @@ -88,6 +91,7 @@ def test_map_scale_kernel(): batch_shape=torch.Size(), ard_num_dims=10, active_dims=list(range(5)), + features_to_idx_mapper=None, ) assert hasattr(k, "outputscale_prior") is False @@ -99,6 +103,7 @@ def test_map_polynomial_kernel(): batch_shape=torch.Size(), ard_num_dims=10, active_dims=list(range(5)), + features_to_idx_mapper=None, ) assert hasattr(k, "offset_prior") assert isinstance(k.offset_prior, gpytorch.priors.GammaPrior) @@ -108,6 +113,7 @@ def test_map_polynomial_kernel(): batch_shape=torch.Size(), ard_num_dims=10, active_dims=list(range(5)), + features_to_idx_mapper=None, ) assert hasattr(k, "offset_prior") is False @@ -163,6 +169,7 @@ def test_map_continuous_kernel(kernel, ard_num_dims, active_dims, expected_kerne batch_shape=torch.Size(), ard_num_dims=ard_num_dims, active_dims=active_dims, + features_to_idx_mapper=None, ) assert isinstance(k, expected_kernel) if isinstance(kernel, LinearKernel): @@ -206,6 +213,7 @@ def test_map_molecular_kernel(kernel, ard_num_dims, active_dims, expected_kernel batch_shape=torch.Size(), ard_num_dims=ard_num_dims, active_dims=active_dims, + features_to_idx_mapper=None, ) assert isinstance(k, expected_kernel) @@ -226,6 +234,7 @@ def test_map_wasserstein_kernel(): batch_shape=torch.Size(), ard_num_dims=10, active_dims=list(range(5)), + features_to_idx_mapper=None, ) assert isinstance(k, shapeKernels.WassersteinKernel) assert hasattr(k, "lengthscale_prior") @@ -237,6 +246,7 @@ def test_map_wasserstein_kernel(): batch_shape=torch.Size(), ard_num_dims=10, active_dims=list(range(5)), + features_to_idx_mapper=None, ) assert k.squared is True assert hasattr(k, "lengthscale_prior") is False diff --git a/tests/bofire/strategies/doe/test_design.py b/tests/bofire/strategies/doe/test_design.py index ab956e809..2bd72422a 100644 --- a/tests/bofire/strategies/doe/test_design.py +++ b/tests/bofire/strategies/doe/test_design.py @@ -664,6 +664,7 @@ def test_fixed_experiments_checker(): def test_partially_fixed_experiments(): + pytest.importorskip("docutils") domain = Domain( inputs=[ ContinuousInput(key="x1", bounds=(0, 5)), diff --git a/tests/bofire/strategies/test_doe.py b/tests/bofire/strategies/test_doe.py index a99c5853b..b3607f3a2 100644 --- a/tests/bofire/strategies/test_doe.py +++ b/tests/bofire/strategies/test_doe.py @@ -24,6 +24,9 @@ warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=UserWarning, append=True) +import pytest +pytest.importorskip("cyipopt") + with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/tests/bofire/strategies/test_from_data_model.py b/tests/bofire/strategies/test_from_data_model.py index ff1873490..9b163a613 100644 --- a/tests/bofire/strategies/test_from_data_model.py +++ b/tests/bofire/strategies/test_from_data_model.py @@ -1,7 +1,10 @@ +import pytest + from bofire.strategies import api as strategies def test_strategy_can_be_loaded_from_data_model(strategy_spec): + pytest.importorskip("entmoot") data_model = strategy_spec.obj() strategy = strategies.map(data_model=data_model) assert strategy is not None diff --git a/tests/bofire/strategies/test_space_filling.py b/tests/bofire/strategies/test_space_filling.py index 46e3676de..43c858713 100644 --- a/tests/bofire/strategies/test_space_filling.py +++ b/tests/bofire/strategies/test_space_filling.py @@ -14,6 +14,9 @@ from bofire.data_models.features.api import ContinuousInput +import pytest +pytest.importorskip("cyipopt") + inputs = [ContinuousInput(key=f"if{i}", bounds=(0, 1)) for i in range(1, 4)] c1 = LinearInequalityConstraint( features=["if1", "if2", "if3"], diff --git a/tests/bofire/surrogates/test_xgb.py b/tests/bofire/surrogates/test_xgb.py index f01b1fd5f..6d56516a4 100644 --- a/tests/bofire/surrogates/test_xgb.py +++ b/tests/bofire/surrogates/test_xgb.py @@ -15,6 +15,9 @@ from bofire.data_models.surrogates.api import XGBoostSurrogate +pytest.importorskip("xgboost") + + XGB_AVAILABLE = importlib.util.find_spec("xgboost") is not None From a806dc06f1c5a8c37c758fe300e1d3e5d8bef254 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Wed, 4 Dec 2024 15:39:45 +0100 Subject: [PATCH 02/43] pre-commit --- bofire/data_models/kernels/aggregation.py | 2 +- bofire/data_models/kernels/categorical.py | 2 +- bofire/data_models/kernels/kernel.py | 2 +- bofire/data_models/kernels/molecular.py | 2 +- tests/bofire/strategies/test_doe.py | 3 ++- tests/bofire/strategies/test_space_filling.py | 1 - 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bofire/data_models/kernels/aggregation.py b/bofire/data_models/kernels/aggregation.py index d7751ef3b..bcc92525d 100644 --- a/bofire/data_models/kernels/aggregation.py +++ b/bofire/data_models/kernels/aggregation.py @@ -3,7 +3,7 @@ from bofire.data_models.kernels.categorical import HammingDistanceKernel from bofire.data_models.kernels.continuous import LinearKernel, MaternKernel, RBFKernel -from bofire.data_models.kernels.kernel import AggregationKernel, Kernel +from bofire.data_models.kernels.kernel import AggregationKernel from bofire.data_models.kernels.molecular import TanimotoKernel from bofire.data_models.kernels.shape import WassersteinKernel from bofire.data_models.priors.api import AnyGeneralPrior diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index e3ef4c5d9..4fa2e0d72 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional +from typing import Literal from bofire.data_models.kernels.kernel import ConcreteKernel diff --git a/bofire/data_models/kernels/kernel.py b/bofire/data_models/kernels/kernel.py index 5c74918cc..c31744aff 100644 --- a/bofire/data_models/kernels/kernel.py +++ b/bofire/data_models/kernels/kernel.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional +from typing import List, Optional from bofire.data_models.base import BaseModel diff --git a/bofire/data_models/kernels/molecular.py b/bofire/data_models/kernels/molecular.py index 5e0a371a1..c30932a40 100644 --- a/bofire/data_models/kernels/molecular.py +++ b/bofire/data_models/kernels/molecular.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional +from typing import Literal from bofire.data_models.kernels.kernel import ConcreteKernel diff --git a/tests/bofire/strategies/test_doe.py b/tests/bofire/strategies/test_doe.py index b3607f3a2..ea50b27cd 100644 --- a/tests/bofire/strategies/test_doe.py +++ b/tests/bofire/strategies/test_doe.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd +import pytest import bofire.data_models.strategies.api as data_models from bofire.data_models.constraints.api import ( @@ -24,7 +25,7 @@ warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.filterwarnings("ignore", category=UserWarning, append=True) -import pytest + pytest.importorskip("cyipopt") with warnings.catch_warnings(): diff --git a/tests/bofire/strategies/test_space_filling.py b/tests/bofire/strategies/test_space_filling.py index 43c858713..e44bd0288 100644 --- a/tests/bofire/strategies/test_space_filling.py +++ b/tests/bofire/strategies/test_space_filling.py @@ -14,7 +14,6 @@ from bofire.data_models.features.api import ContinuousInput -import pytest pytest.importorskip("cyipopt") inputs = [ContinuousInput(key=f"if{i}", bounds=(0, 1)) for i in range(1, 4)] From c3f1899da5e530bc7d8932b763dc07452bfc74c1 Mon Sep 17 00:00:00 2001 From: Robert Lee <84771576+R-M-Lee@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:33:24 +0100 Subject: [PATCH 03/43] test map singletaskgp with additive kernel --- tests/bofire/surrogates/test_gps.py | 42 ++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index 1dc812dfd..f9630923b 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -15,7 +15,7 @@ from pydantic import ValidationError import bofire.surrogates.api as surrogates -from bofire.benchmarks.api import Himmelblau +from bofire.benchmarks.api import Hartmann, Himmelblau from bofire.data_models.domain.api import Inputs, Outputs from bofire.data_models.enum import CategoricalEncodingEnum, RegressionMetricsEnum from bofire.data_models.features.api import ( @@ -25,6 +25,7 @@ MolecularInput, ) from bofire.data_models.kernels.api import ( + AdditiveKernel, HammingDistanceKernel, MaternKernel, RBFKernel, @@ -293,6 +294,45 @@ def test_SingleTaskGPHyperconfig(): ) +def test_SingleTaskGPModel_feature_subsets(): + """make an additive kernel using feature subsets for each kernel in the sum""" + benchmark = Hartmann() + bench_x = benchmark.domain.inputs.sample(12) + bench_expts = pd.concat([bench_x, benchmark.f(bench_x)], axis=1) + + input_names = benchmark.domain.inputs.get_keys() + inputs_kernel_1 = input_names[:2] + inputs_kernel_2 = input_names[2:] + + gp_data = SingleTaskGPSurrogate( + inputs=benchmark.domain.inputs, + outputs=benchmark.domain.outputs, + kernel=AdditiveKernel( + kernels=[ + RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=inputs_kernel_1, + ), + RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=inputs_kernel_2, + ), + ] + ), + ) + + gp_mapped = surrogates.map(gp_data) + assert hasattr(gp_mapped, "fit") + assert len(gp_mapped.kernel.kernels) == 2 + assert gp_mapped.kernel.kernels[0].features == ["x_0", "x_1"] + assert gp_mapped.kernel.kernels[1].features == ["x_2", "x_3", "x_4", "x_5"] + gp_mapped.fit(bench_expts) + pred = gp_mapped.predict(bench_expts) + assert pred.shape == (12, 2) + + def test_MixedSingleTaskGPHyperconfig(): inputs = Inputs( features=[ From 86b6ad678e605e647b9f36327d1fe5bcef73c83a Mon Sep 17 00:00:00 2001 From: Robert Lee <84771576+R-M-Lee@users.noreply.github.com> Date: Wed, 4 Dec 2024 16:50:17 +0100 Subject: [PATCH 04/43] test active_dims of mapped kernels --- tests/bofire/surrogates/test_gps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index f9630923b..759aae261 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -331,6 +331,8 @@ def test_SingleTaskGPModel_feature_subsets(): gp_mapped.fit(bench_expts) pred = gp_mapped.predict(bench_expts) assert pred.shape == (12, 2) + assert len(gp_mapped.model.covar_module.kernels[0].active_dims) == 2 + assert len(gp_mapped.model.covar_module.kernels[1].active_dims) == 4 def test_MixedSingleTaskGPHyperconfig(): From b4458fbfabfa6d22851ce76b6110f813cc8a73cb Mon Sep 17 00:00:00 2001 From: Robert Lee <84771576+R-M-Lee@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:00:53 +0100 Subject: [PATCH 05/43] add features_to_idx_mapper to outlier detection tutorial --- tutorials/benchmarks/007-Benchmark_outlier_detection.ipynb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tutorials/benchmarks/007-Benchmark_outlier_detection.ipynb b/tutorials/benchmarks/007-Benchmark_outlier_detection.ipynb index 1034aa19d..8dd5e587c 100644 --- a/tutorials/benchmarks/007-Benchmark_outlier_detection.ipynb +++ b/tutorials/benchmarks/007-Benchmark_outlier_detection.ipynb @@ -208,6 +208,9 @@ " batch_shape=torch.Size(),\n", " active_dims=list(range(tX.shape[1])),\n", " ard_num_dims=1, # this keyword is ignored\n", + " features_to_idx_mapper=lambda feats: self.inputs.get_feature_indices(\n", + " self.input_preprocessing_specs, feats\n", + " ),\n", " ),\n", " # outcome_transform=Standardize(m=tY.shape[-1]),\n", " input_transform=scaler,\n", @@ -767,7 +770,7 @@ ], "metadata": { "kernelspec": { - "display_name": "base", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -781,7 +784,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.10.12" }, "papermill": { "default_parameters": {}, From d243ad022ab1323ee6a5ba81974d6f1f6954f96c Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Wed, 4 Dec 2024 17:11:43 +0100 Subject: [PATCH 06/43] correctly handling categorical mol features --- bofire/strategies/predictives/botorch.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bofire/strategies/predictives/botorch.py b/bofire/strategies/predictives/botorch.py index 71c48c67e..43bc7c779 100644 --- a/bofire/strategies/predictives/botorch.py +++ b/bofire/strategies/predictives/botorch.py @@ -27,6 +27,7 @@ from bofire.data_models.features.api import ( CategoricalDescriptorInput, CategoricalInput, + CategoricalMolecularInput, DiscreteInput, Input, ) @@ -652,6 +653,12 @@ def get_categorical_combinations(self) -> List[Dict[int, float]]: for j, idx in enumerate(features2idx[feat]): fixed_features[idx] = feature.values[index][j] + elif isinstance(feature, CategoricalMolecularInput): + transformed = feature.to_descriptor_encoding( + self.input_preprocessing_specs[feat], pd.Series([val]) + ) + for j, idx in enumerate(features2idx[feat]): + fixed_features[idx] = transformed.values[0, j] elif isinstance(feature, CategoricalInput): # it has to be onehot in this case transformed = feature.to_onehot_encoding(pd.Series([val])) From 842797f1faf2eeae80bceefd32b8975693b86345 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Wed, 4 Dec 2024 17:32:12 +0100 Subject: [PATCH 07/43] validating mol features transforms --- bofire/data_models/domain/features.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bofire/data_models/domain/features.py b/bofire/data_models/domain/features.py index d4b21c4b8..57b01b2d0 100644 --- a/bofire/data_models/domain/features.py +++ b/bofire/data_models/domain/features.py @@ -593,6 +593,7 @@ def _validate_transform_specs( """ # first check that the keys in the specs dict are correct also correct feature keys # next check that all values are of type CategoricalEncodingEnum or MolFeatures + checked_keys = set() for key, value in specs.items(): try: feat = self.get_by_key(key) @@ -622,6 +623,21 @@ def _validate_transform_specs( raise ValueError( f"Forbidden transform type for feature with key {key}", ) + checked_keys.add(key) + + # now check that features that must be transformed do have a transformation defined + for key in self.get_keys(): + if key in checked_keys: + continue + + feat = self.get_by_key(key) + if isinstance(feat, MolecularInput): + trx = specs.get(key) + if trx is None or not isinstance(trx, MolFeatures): + raise ValueError( + "MolecularInput features must have a input processing of type MolFeatures defined" + ) + return specs def get_bounds( From 22c438265c22bc4208c0f2f901d03b05810413b1 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Fri, 6 Dec 2024 09:48:13 +0100 Subject: [PATCH 08/43] verifying proper type --- bofire/strategies/predictives/botorch.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bofire/strategies/predictives/botorch.py b/bofire/strategies/predictives/botorch.py index 43bc7c779..d819864db 100644 --- a/bofire/strategies/predictives/botorch.py +++ b/bofire/strategies/predictives/botorch.py @@ -31,6 +31,7 @@ DiscreteInput, Input, ) +from bofire.data_models.molfeatures.api import AnyMolFeatures from bofire.data_models.strategies.api import BotorchStrategy as DataModel from bofire.data_models.strategies.api import RandomStrategy as RandomStrategyDataModel from bofire.data_models.strategies.api import ( @@ -654,8 +655,13 @@ def get_categorical_combinations(self) -> List[Dict[int, float]]: fixed_features[idx] = feature.values[index][j] elif isinstance(feature, CategoricalMolecularInput): + preproc = self.input_preprocessing_specs[feat] + if not isinstance(preproc, AnyMolFeatures): + raise ValueError( + f"preprocessing for {feat} must be of type AnyMolFeatures" + ) transformed = feature.to_descriptor_encoding( - self.input_preprocessing_specs[feat], pd.Series([val]) + preproc, pd.Series([val]) ) for j, idx in enumerate(features2idx[feat]): fixed_features[idx] = transformed.values[0, j] From 17d8350a20cfb79182f166a4027306f338843883 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 11:21:44 +0100 Subject: [PATCH 09/43] custom hamming kernel enabling single task gp on categorical features --- bofire/data_models/kernels/categorical.py | 3 +- bofire/kernels/categorical.py | 25 ++++ bofire/kernels/mapper.py | 35 +++++- scratch.py | 132 ++++++++++++++++++++++ tests/bofire/surrogates/test_gps.py | 55 +++++++++ 5 files changed, 243 insertions(+), 7 deletions(-) create mode 100644 bofire/kernels/categorical.py create mode 100644 scratch.py diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index 4fa2e0d72..8d03c429d 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Literal, Optional from bofire.data_models.kernels.kernel import ConcreteKernel @@ -10,3 +10,4 @@ class CategoricalKernel(ConcreteKernel): class HammingDistanceKernel(CategoricalKernel): type: Literal["HammingDistanceKernel"] = "HammingDistanceKernel" ard: bool = True + with_one_hots: Optional[bool] = None diff --git a/bofire/kernels/categorical.py b/bofire/kernels/categorical.py new file mode 100644 index 000000000..7e04065de --- /dev/null +++ b/bofire/kernels/categorical.py @@ -0,0 +1,25 @@ +import torch +from gpytorch.kernels.kernel import Kernel +from torch import Tensor + + +class HammingKernelWithOneHots(Kernel): + has_lengthscale = True + + def forward( + self, + x1: Tensor, + x2: Tensor, + diag: bool = False, + last_dim_is_batch: bool = False, + ) -> Tensor: + delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3))**2 + dists = delta / self.lengthscale.unsqueeze(-2) + if last_dim_is_batch: + dists = dists.transpose(-3, -1) + + dists = dists.sum(-1) / 2 + res = torch.exp(-dists) + if diag: + res = torch.diagonal(res, dim1=-1, dim2=-2) + return res diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index f05baf790..7d860963e 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -7,6 +7,7 @@ import bofire.data_models.kernels.api as data_models import bofire.priors.api as priors +from bofire.kernels.categorical import HammingKernelWithOneHots from bofire.kernels.fingerprint_kernels.tanimoto_kernel import TanimotoKernel from bofire.kernels.shape import WassersteinKernel @@ -215,13 +216,35 @@ def map_HammingDistanceKernel( ard_num_dims: int, active_dims: List[int], features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], -) -> CategoricalKernel: +) -> GpytorchKernel: active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) - return CategoricalKernel( - batch_shape=batch_shape, - ard_num_dims=len(active_dims) if data_model.ard else None, - active_dims=active_dims, # type: ignore - ) + + if data_model.with_one_hots is None: + with_one_hots = data_model.features is not None and len(active_dims) > 1 + else: + with_one_hots = data_model.with_one_hots + + if with_one_hots and len(active_dims) == 1: + raise RuntimeError( + "only one feature for categorical kernel operating on one-hot features" + ) + elif not with_one_hots and len(active_dims) > 1: + # this is not necessarily an issue since botorch's CategoricalKernel + # can work on multiple features at the same time + pass + + if with_one_hots: + return HammingKernelWithOneHots( + batch_shape=batch_shape, + ard_num_dims=len(active_dims) if data_model.ard else None, + active_dims=active_dims, # type: ignore + ) + else: + return CategoricalKernel( + batch_shape=batch_shape, + ard_num_dims=len(active_dims) if data_model.ard else None, + active_dims=active_dims, # type: ignore + ) def map_WassersteinKernel( diff --git a/scratch.py b/scratch.py new file mode 100644 index 000000000..15caab666 --- /dev/null +++ b/scratch.py @@ -0,0 +1,132 @@ +import pandas as pd + +import bofire.strategies.api as strategies +import bofire.surrogates.api as surrogates +from bofire.data_models.domain import api as domain_api +from bofire.data_models.features import api as features_api +from bofire.data_models.kernels import api as kernels_api +from bofire.data_models.molfeatures import api as molfeatures_api +from bofire.data_models.priors.api import HVARFNER_LENGTHSCALE_PRIOR +from bofire.data_models.strategies import api as strategies_api +from bofire.data_models.surrogates import api as surrogates_api + + +def test_SingleTaskGPModel_mixed_features(): + """test that we can use a single task gp with mixed features""" + inputs = domain_api.Inputs( + features=[ + features_api.ContinuousInput( + key=f"x_{i+1}", + bounds=(-4, 4), + ) + for i in range(2) + ] + + [ + features_api.CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), + features_api.CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), + ] + ) + outputs = domain_api.Outputs(features=[features_api.ContinuousOutput(key="y")]) + experiments = inputs.sample(n=10) + experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) + experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 + experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 + experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 + experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 + experiments["valid_y"] = 1 + + gp_data = surrogates_api.SingleTaskGPSurrogate( + inputs=inputs, + outputs=outputs, + kernel=kernels_api.AdditiveKernel( + kernels=[ + kernels_api.HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + kernels_api.RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=[f"x_{i+1}" for i in range(2)], + ), + ] + ), + ) + + gp_mapped = surrogates.map(gp_data) + assert hasattr(gp_mapped, "fit") + assert len(gp_mapped.kernel.kernels) == 2 + assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] + assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] + gp_mapped.fit(experiments) + pred = gp_mapped.predict(experiments) + assert pred.shape == (10, 2) + assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] + assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] + + +if __name__ == "__main__": + test_SingleTaskGPModel_mixed_features() + + +import sys + + +sys.exit(0) + + +domain = domain_api.Domain( + inputs=domain_api.Inputs( + features=[ + features_api.ContinuousInput(key="x1", bounds=(-1, 1)), + features_api.ContinuousInput(key="x2", bounds=(-1, 1)), + features_api.CategoricalMolecularInput( + key="mol", categories=["CO", "CCO", "CCCO"] + ), + ] + ), + outputs=domain_api.Outputs(features=[features_api.ContinuousOutput(key="f")]), +) + + +strategy = strategies.map( + strategies_api.SoboStrategy( + domain=domain, + surrogate_specs=surrogates_api.BotorchSurrogates( + surrogates=[ + surrogates_api.SingleTaskGPSurrogate( + inputs=domain.inputs, + outputs=domain.outputs, + input_preprocessing_specs={ + "mol": molfeatures_api.Fingerprints(), + }, + kernel=kernels_api.AdditiveKernel( + kernels=[ + kernels_api.RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=["x1", "x2"], + ), + kernels_api.TanimotoKernel( + features=["mol"], + ), + ] + ), + ) + ] + ), + ) +) + + +strategy.tell( + experiments=pd.DataFrame( + [ + {"x1": 0.2, "x2": 0.4, "mol": "CO", "f": 1.0}, + {"x1": 0.4, "x2": 0.2, "mol": "CCO", "f": 2.0}, + {"x1": 0.6, "x2": 0.6, "mol": "CCCO", "f": 3.0}, + ] + ) +) +candidates = strategy.ask(candidate_count=1) +print(candidates) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index 759aae261..d2dbd2861 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -335,6 +335,61 @@ def test_SingleTaskGPModel_feature_subsets(): assert len(gp_mapped.model.covar_module.kernels[1].active_dims) == 4 +def test_SingleTaskGPModel_mixed_features(): + """test that we can use a single task gp with mixed features""" + inputs = Inputs( + features=[ + ContinuousInput( + key=f"x_{i+1}", + bounds=(-4, 4), + ) + for i in range(2) + ] + + [ + CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), + CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), + ], + ) + outputs = Outputs(features=[ContinuousOutput(key="y")]) + experiments = inputs.sample(n=10) + experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) + experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 + experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 + experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 + experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 + experiments["valid_y"] = 1 + + gp_data = SingleTaskGPSurrogate( + inputs=inputs, + outputs=outputs, + kernel=AdditiveKernel( + kernels=[ + HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=[f"x_{i+1}" for i in range(2)], + ), + ] + ), + ) + + gp_mapped = surrogates.map(gp_data) + assert hasattr(gp_mapped, "fit") + assert len(gp_mapped.kernel.kernels) == 2 + assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] + assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] + gp_mapped.fit(experiments) + pred = gp_mapped.predict(experiments) + assert pred.shape == (10, 2) + assert ((pred['y_pred'] - experiments['y'])**2).mean() < 0.5 + assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] + assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] + + def test_MixedSingleTaskGPHyperconfig(): inputs = Inputs( features=[ From 6ad1dfd29722a0d6e2d58eb94fd8b491e68dc2a0 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 11:52:49 +0100 Subject: [PATCH 10/43] removed unnecessary parameter from data model --- bofire/data_models/kernels/categorical.py | 3 +- bofire/kernels/categorical.py | 2 +- bofire/kernels/mapper.py | 6 +- scratch.py | 132 ---------------------- tests/bofire/surrogates/test_gps.py | 4 +- 5 files changed, 5 insertions(+), 142 deletions(-) delete mode 100644 scratch.py diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index 8d03c429d..4fa2e0d72 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -1,4 +1,4 @@ -from typing import Literal, Optional +from typing import Literal from bofire.data_models.kernels.kernel import ConcreteKernel @@ -10,4 +10,3 @@ class CategoricalKernel(ConcreteKernel): class HammingDistanceKernel(CategoricalKernel): type: Literal["HammingDistanceKernel"] = "HammingDistanceKernel" ard: bool = True - with_one_hots: Optional[bool] = None diff --git a/bofire/kernels/categorical.py b/bofire/kernels/categorical.py index 7e04065de..640a62d1e 100644 --- a/bofire/kernels/categorical.py +++ b/bofire/kernels/categorical.py @@ -13,7 +13,7 @@ def forward( diag: bool = False, last_dim_is_batch: bool = False, ) -> Tensor: - delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3))**2 + delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3)) ** 2 dists = delta / self.lengthscale.unsqueeze(-2) if last_dim_is_batch: dists = dists.transpose(-3, -1) diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index 7d860963e..1425d35f1 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -219,11 +219,7 @@ def map_HammingDistanceKernel( ) -> GpytorchKernel: active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) - if data_model.with_one_hots is None: - with_one_hots = data_model.features is not None and len(active_dims) > 1 - else: - with_one_hots = data_model.with_one_hots - + with_one_hots = data_model.features is not None and len(active_dims) > 1 if with_one_hots and len(active_dims) == 1: raise RuntimeError( "only one feature for categorical kernel operating on one-hot features" diff --git a/scratch.py b/scratch.py deleted file mode 100644 index 15caab666..000000000 --- a/scratch.py +++ /dev/null @@ -1,132 +0,0 @@ -import pandas as pd - -import bofire.strategies.api as strategies -import bofire.surrogates.api as surrogates -from bofire.data_models.domain import api as domain_api -from bofire.data_models.features import api as features_api -from bofire.data_models.kernels import api as kernels_api -from bofire.data_models.molfeatures import api as molfeatures_api -from bofire.data_models.priors.api import HVARFNER_LENGTHSCALE_PRIOR -from bofire.data_models.strategies import api as strategies_api -from bofire.data_models.surrogates import api as surrogates_api - - -def test_SingleTaskGPModel_mixed_features(): - """test that we can use a single task gp with mixed features""" - inputs = domain_api.Inputs( - features=[ - features_api.ContinuousInput( - key=f"x_{i+1}", - bounds=(-4, 4), - ) - for i in range(2) - ] - + [ - features_api.CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), - features_api.CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), - ] - ) - outputs = domain_api.Outputs(features=[features_api.ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10) - experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) - experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 - experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 - experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 - experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 - experiments["valid_y"] = 1 - - gp_data = surrogates_api.SingleTaskGPSurrogate( - inputs=inputs, - outputs=outputs, - kernel=kernels_api.AdditiveKernel( - kernels=[ - kernels_api.HammingDistanceKernel( - ard=True, - features=["x_cat_1", "x_cat_2"], - ), - kernels_api.RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=[f"x_{i+1}" for i in range(2)], - ), - ] - ), - ) - - gp_mapped = surrogates.map(gp_data) - assert hasattr(gp_mapped, "fit") - assert len(gp_mapped.kernel.kernels) == 2 - assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] - assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] - gp_mapped.fit(experiments) - pred = gp_mapped.predict(experiments) - assert pred.shape == (10, 2) - assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] - assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] - - -if __name__ == "__main__": - test_SingleTaskGPModel_mixed_features() - - -import sys - - -sys.exit(0) - - -domain = domain_api.Domain( - inputs=domain_api.Inputs( - features=[ - features_api.ContinuousInput(key="x1", bounds=(-1, 1)), - features_api.ContinuousInput(key="x2", bounds=(-1, 1)), - features_api.CategoricalMolecularInput( - key="mol", categories=["CO", "CCO", "CCCO"] - ), - ] - ), - outputs=domain_api.Outputs(features=[features_api.ContinuousOutput(key="f")]), -) - - -strategy = strategies.map( - strategies_api.SoboStrategy( - domain=domain, - surrogate_specs=surrogates_api.BotorchSurrogates( - surrogates=[ - surrogates_api.SingleTaskGPSurrogate( - inputs=domain.inputs, - outputs=domain.outputs, - input_preprocessing_specs={ - "mol": molfeatures_api.Fingerprints(), - }, - kernel=kernels_api.AdditiveKernel( - kernels=[ - kernels_api.RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=["x1", "x2"], - ), - kernels_api.TanimotoKernel( - features=["mol"], - ), - ] - ), - ) - ] - ), - ) -) - - -strategy.tell( - experiments=pd.DataFrame( - [ - {"x1": 0.2, "x2": 0.4, "mol": "CO", "f": 1.0}, - {"x1": 0.4, "x2": 0.2, "mol": "CCO", "f": 2.0}, - {"x1": 0.6, "x2": 0.6, "mol": "CCCO", "f": 3.0}, - ] - ) -) -candidates = strategy.ask(candidate_count=1) -print(candidates) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index d2dbd2861..8f43c3ac4 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -351,7 +351,7 @@ def test_SingleTaskGPModel_mixed_features(): ], ) outputs = Outputs(features=[ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10) + experiments = inputs.sample(n=10, seed=194387) experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 @@ -385,7 +385,7 @@ def test_SingleTaskGPModel_mixed_features(): gp_mapped.fit(experiments) pred = gp_mapped.predict(experiments) assert pred.shape == (10, 2) - assert ((pred['y_pred'] - experiments['y'])**2).mean() < 0.5 + assert ((pred["y_pred"] - experiments["y"]) ** 2).mean() < 0.5 assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] From 4a2a54766cf3d433e490c5cd1c15d5bee907cb41 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 13:02:49 +0100 Subject: [PATCH 11/43] testing equivalence of mixed gp and single gp with custom kernel --- tests/bofire/surrogates/test_gps.py | 89 +++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index 8f43c3ac4..695cf70a6 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -28,6 +28,7 @@ AdditiveKernel, HammingDistanceKernel, MaternKernel, + MultiplicativeKernel, RBFKernel, ScaleKernel, ) @@ -617,3 +618,91 @@ def test_MixedSingleTaskGPModel_mordred(kernel, scaler, output_scaler): model2.loads(dump) preds2 = model2.predict(experiments.iloc[:-1]) assert_frame_equal(preds, preds2) + + +def test_SingleTaskGP_with_categoricals_is_equivalent_to_MixedSingleTaskGP(): + inputs = Inputs( + features=[ + ContinuousInput( + key=f"x_{i+1}", + bounds=(-4, 4), + ) + for i in range(2) + ] + + [ + CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), + CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), + ] + ) + outputs = Outputs(features=[ContinuousOutput(key="y")]) + experiments = inputs.sample(n=10, seed=194387) + experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) + experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 + experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 + experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 + experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 + experiments["valid_y"] = 1 + + gp_data = SingleTaskGPSurrogate( + inputs=inputs, + outputs=outputs, + kernel=AdditiveKernel( # use the same kernel as botorch.models.MixedSingleTaskGP + kernels=[ + ScaleKernel( + base_kernel=HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ) + ), + ScaleKernel( + base_kernel=RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=[f"x_{i+1}" for i in range(2)], + ) + ), + ScaleKernel( + base_kernel=MultiplicativeKernel( + kernels=[ + HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=[f"x_{i+1}" for i in range(2)], + ), + ] + ) + ), + ] + ), + ) + + gp_mapped = surrogates.map(gp_data) + gp_mapped.fit(experiments) + pred = gp_mapped.predict(experiments) + single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() + assert single_task_gp_mse < 3.5 + + model = MixedSingleTaskGPSurrogate( + inputs=inputs, + outputs=outputs, + input_preprocessing_specs={ + "x_cat_1": CategoricalEncodingEnum.ONE_HOT, + "x_cat_2": CategoricalEncodingEnum.ONE_HOT, + }, + continuous_kernel=RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + ), + categorical_kernel=HammingDistanceKernel(ard=True), + ) + model = surrogates.map(model) + model.fit(experiments) + pred = model.predict(experiments) + mixed_single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() + assert mixed_single_task_gp_mse < 3.5 + + assert abs(single_task_gp_mse - mixed_single_task_gp_mse) < 0.005 From 3750827b62dd02a1305184cbb6e63cf57063d990 Mon Sep 17 00:00:00 2001 From: Emilio Dorigatti Date: Thu, 19 Dec 2024 14:18:10 +0100 Subject: [PATCH 12/43] (temporary) running on all py versions --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 56e950611..a2dcfd627 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -40,6 +40,7 @@ jobs: testing: runs-on: ubuntu-latest + continue-on-error: true strategy: matrix: python-version: [ '3.9', '3.11' ] From 71629838e0cca18c8c0bfbb2da25b44f01eb8a90 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 14:45:13 +0100 Subject: [PATCH 13/43] (temporary) debug github actions by printing --- tests/bofire/strategies/doe/test_objective.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/bofire/strategies/doe/test_objective.py b/tests/bofire/strategies/doe/test_objective.py index 96bd66723..b468a1f1b 100644 --- a/tests/bofire/strategies/doe/test_objective.py +++ b/tests/bofire/strategies/doe/test_objective.py @@ -44,7 +44,12 @@ def test_Objective_model_jacobian_t(): B[:, 4] = np.array([0, 0, 6]) B[:, 5] = np.array([2, 1, 0]) - assert np.allclose(B, model_jacobian_t(x)) + J = model_jacobian_t(x) + print('===== B\n', B) + print('===== J\n', J) + print('===== Jterms\n', objective.terms_jacobian_t) + + assert np.allclose(B, J) # fully quadratic model f = Formula("x1 + x2 + x3 + x1:x2 + x1:x3 + x2:x3 + {x1**2} + {x2**2} + {x3**2}") From 01a01e13681c840a18359412e791df00b6bd6385 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 15:06:57 +0100 Subject: [PATCH 14/43] more printing --- bofire/strategies/doe/objective.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bofire/strategies/doe/objective.py b/bofire/strategies/doe/objective.py index 973a5e7de..473cb19fa 100644 --- a/bofire/strategies/doe/objective.py +++ b/bofire/strategies/doe/objective.py @@ -54,7 +54,9 @@ def __init__( # terms for model jacobian self.terms_jacobian_t = [] + print('model', model) for var in self.vars: + print('diff', var, model.differentiate(var, use_sympy=True)) _terms = [ str(term).replace(":", "*") + f" + 0 * {self.vars[0]}" for term in model.differentiate(var, use_sympy=True) From 1cd2776d80a44ff11c0329da1f979c70e0298521 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 15:32:00 +0100 Subject: [PATCH 15/43] Revert "testing equivalence of mixed gp and single gp with custom kernel" This reverts commit 4a2a54766cf3d433e490c5cd1c15d5bee907cb41. --- tests/bofire/surrogates/test_gps.py | 89 ----------------------------- 1 file changed, 89 deletions(-) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index 695cf70a6..8f43c3ac4 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -28,7 +28,6 @@ AdditiveKernel, HammingDistanceKernel, MaternKernel, - MultiplicativeKernel, RBFKernel, ScaleKernel, ) @@ -618,91 +617,3 @@ def test_MixedSingleTaskGPModel_mordred(kernel, scaler, output_scaler): model2.loads(dump) preds2 = model2.predict(experiments.iloc[:-1]) assert_frame_equal(preds, preds2) - - -def test_SingleTaskGP_with_categoricals_is_equivalent_to_MixedSingleTaskGP(): - inputs = Inputs( - features=[ - ContinuousInput( - key=f"x_{i+1}", - bounds=(-4, 4), - ) - for i in range(2) - ] - + [ - CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), - CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), - ] - ) - outputs = Outputs(features=[ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10, seed=194387) - experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) - experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 - experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 - experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 - experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 - experiments["valid_y"] = 1 - - gp_data = SingleTaskGPSurrogate( - inputs=inputs, - outputs=outputs, - kernel=AdditiveKernel( # use the same kernel as botorch.models.MixedSingleTaskGP - kernels=[ - ScaleKernel( - base_kernel=HammingDistanceKernel( - ard=True, - features=["x_cat_1", "x_cat_2"], - ) - ), - ScaleKernel( - base_kernel=RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=[f"x_{i+1}" for i in range(2)], - ) - ), - ScaleKernel( - base_kernel=MultiplicativeKernel( - kernels=[ - HammingDistanceKernel( - ard=True, - features=["x_cat_1", "x_cat_2"], - ), - RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=[f"x_{i+1}" for i in range(2)], - ), - ] - ) - ), - ] - ), - ) - - gp_mapped = surrogates.map(gp_data) - gp_mapped.fit(experiments) - pred = gp_mapped.predict(experiments) - single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() - assert single_task_gp_mse < 3.5 - - model = MixedSingleTaskGPSurrogate( - inputs=inputs, - outputs=outputs, - input_preprocessing_specs={ - "x_cat_1": CategoricalEncodingEnum.ONE_HOT, - "x_cat_2": CategoricalEncodingEnum.ONE_HOT, - }, - continuous_kernel=RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - ), - categorical_kernel=HammingDistanceKernel(ard=True), - ) - model = surrogates.map(model) - model.fit(experiments) - pred = model.predict(experiments) - mixed_single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() - assert mixed_single_task_gp_mse < 3.5 - - assert abs(single_task_gp_mse - mixed_single_task_gp_mse) < 0.005 From 8400fdbc94a1dd4eee254daa7dede618e60760d8 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 15:32:19 +0100 Subject: [PATCH 16/43] Revert "removed unnecessary parameter from data model" This reverts commit 6ad1dfd29722a0d6e2d58eb94fd8b491e68dc2a0. --- bofire/data_models/kernels/categorical.py | 3 +- bofire/kernels/categorical.py | 2 +- bofire/kernels/mapper.py | 6 +- scratch.py | 132 ++++++++++++++++++++++ tests/bofire/surrogates/test_gps.py | 4 +- 5 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 scratch.py diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index 4fa2e0d72..8d03c429d 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Literal, Optional from bofire.data_models.kernels.kernel import ConcreteKernel @@ -10,3 +10,4 @@ class CategoricalKernel(ConcreteKernel): class HammingDistanceKernel(CategoricalKernel): type: Literal["HammingDistanceKernel"] = "HammingDistanceKernel" ard: bool = True + with_one_hots: Optional[bool] = None diff --git a/bofire/kernels/categorical.py b/bofire/kernels/categorical.py index 640a62d1e..7e04065de 100644 --- a/bofire/kernels/categorical.py +++ b/bofire/kernels/categorical.py @@ -13,7 +13,7 @@ def forward( diag: bool = False, last_dim_is_batch: bool = False, ) -> Tensor: - delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3)) ** 2 + delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3))**2 dists = delta / self.lengthscale.unsqueeze(-2) if last_dim_is_batch: dists = dists.transpose(-3, -1) diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index 1425d35f1..7d860963e 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -219,7 +219,11 @@ def map_HammingDistanceKernel( ) -> GpytorchKernel: active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) - with_one_hots = data_model.features is not None and len(active_dims) > 1 + if data_model.with_one_hots is None: + with_one_hots = data_model.features is not None and len(active_dims) > 1 + else: + with_one_hots = data_model.with_one_hots + if with_one_hots and len(active_dims) == 1: raise RuntimeError( "only one feature for categorical kernel operating on one-hot features" diff --git a/scratch.py b/scratch.py new file mode 100644 index 000000000..15caab666 --- /dev/null +++ b/scratch.py @@ -0,0 +1,132 @@ +import pandas as pd + +import bofire.strategies.api as strategies +import bofire.surrogates.api as surrogates +from bofire.data_models.domain import api as domain_api +from bofire.data_models.features import api as features_api +from bofire.data_models.kernels import api as kernels_api +from bofire.data_models.molfeatures import api as molfeatures_api +from bofire.data_models.priors.api import HVARFNER_LENGTHSCALE_PRIOR +from bofire.data_models.strategies import api as strategies_api +from bofire.data_models.surrogates import api as surrogates_api + + +def test_SingleTaskGPModel_mixed_features(): + """test that we can use a single task gp with mixed features""" + inputs = domain_api.Inputs( + features=[ + features_api.ContinuousInput( + key=f"x_{i+1}", + bounds=(-4, 4), + ) + for i in range(2) + ] + + [ + features_api.CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), + features_api.CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), + ] + ) + outputs = domain_api.Outputs(features=[features_api.ContinuousOutput(key="y")]) + experiments = inputs.sample(n=10) + experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) + experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 + experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 + experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 + experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 + experiments["valid_y"] = 1 + + gp_data = surrogates_api.SingleTaskGPSurrogate( + inputs=inputs, + outputs=outputs, + kernel=kernels_api.AdditiveKernel( + kernels=[ + kernels_api.HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + kernels_api.RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=[f"x_{i+1}" for i in range(2)], + ), + ] + ), + ) + + gp_mapped = surrogates.map(gp_data) + assert hasattr(gp_mapped, "fit") + assert len(gp_mapped.kernel.kernels) == 2 + assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] + assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] + gp_mapped.fit(experiments) + pred = gp_mapped.predict(experiments) + assert pred.shape == (10, 2) + assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] + assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] + + +if __name__ == "__main__": + test_SingleTaskGPModel_mixed_features() + + +import sys + + +sys.exit(0) + + +domain = domain_api.Domain( + inputs=domain_api.Inputs( + features=[ + features_api.ContinuousInput(key="x1", bounds=(-1, 1)), + features_api.ContinuousInput(key="x2", bounds=(-1, 1)), + features_api.CategoricalMolecularInput( + key="mol", categories=["CO", "CCO", "CCCO"] + ), + ] + ), + outputs=domain_api.Outputs(features=[features_api.ContinuousOutput(key="f")]), +) + + +strategy = strategies.map( + strategies_api.SoboStrategy( + domain=domain, + surrogate_specs=surrogates_api.BotorchSurrogates( + surrogates=[ + surrogates_api.SingleTaskGPSurrogate( + inputs=domain.inputs, + outputs=domain.outputs, + input_preprocessing_specs={ + "mol": molfeatures_api.Fingerprints(), + }, + kernel=kernels_api.AdditiveKernel( + kernels=[ + kernels_api.RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=["x1", "x2"], + ), + kernels_api.TanimotoKernel( + features=["mol"], + ), + ] + ), + ) + ] + ), + ) +) + + +strategy.tell( + experiments=pd.DataFrame( + [ + {"x1": 0.2, "x2": 0.4, "mol": "CO", "f": 1.0}, + {"x1": 0.4, "x2": 0.2, "mol": "CCO", "f": 2.0}, + {"x1": 0.6, "x2": 0.6, "mol": "CCCO", "f": 3.0}, + ] + ) +) +candidates = strategy.ask(candidate_count=1) +print(candidates) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index 8f43c3ac4..d2dbd2861 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -351,7 +351,7 @@ def test_SingleTaskGPModel_mixed_features(): ], ) outputs = Outputs(features=[ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10, seed=194387) + experiments = inputs.sample(n=10) experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 @@ -385,7 +385,7 @@ def test_SingleTaskGPModel_mixed_features(): gp_mapped.fit(experiments) pred = gp_mapped.predict(experiments) assert pred.shape == (10, 2) - assert ((pred["y_pred"] - experiments["y"]) ** 2).mean() < 0.5 + assert ((pred['y_pred'] - experiments['y'])**2).mean() < 0.5 assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] From 2e2985246cfd35cadeef4af4d836e906d259c0c3 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 15:32:34 +0100 Subject: [PATCH 17/43] Revert "custom hamming kernel enabling single task gp on categorical features" This reverts commit 17d8350a20cfb79182f166a4027306f338843883. --- bofire/data_models/kernels/categorical.py | 3 +- bofire/kernels/categorical.py | 25 ---- bofire/kernels/mapper.py | 35 +----- scratch.py | 132 ---------------------- tests/bofire/surrogates/test_gps.py | 55 --------- 5 files changed, 7 insertions(+), 243 deletions(-) delete mode 100644 bofire/kernels/categorical.py delete mode 100644 scratch.py diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index 8d03c429d..4fa2e0d72 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -1,4 +1,4 @@ -from typing import Literal, Optional +from typing import Literal from bofire.data_models.kernels.kernel import ConcreteKernel @@ -10,4 +10,3 @@ class CategoricalKernel(ConcreteKernel): class HammingDistanceKernel(CategoricalKernel): type: Literal["HammingDistanceKernel"] = "HammingDistanceKernel" ard: bool = True - with_one_hots: Optional[bool] = None diff --git a/bofire/kernels/categorical.py b/bofire/kernels/categorical.py deleted file mode 100644 index 7e04065de..000000000 --- a/bofire/kernels/categorical.py +++ /dev/null @@ -1,25 +0,0 @@ -import torch -from gpytorch.kernels.kernel import Kernel -from torch import Tensor - - -class HammingKernelWithOneHots(Kernel): - has_lengthscale = True - - def forward( - self, - x1: Tensor, - x2: Tensor, - diag: bool = False, - last_dim_is_batch: bool = False, - ) -> Tensor: - delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3))**2 - dists = delta / self.lengthscale.unsqueeze(-2) - if last_dim_is_batch: - dists = dists.transpose(-3, -1) - - dists = dists.sum(-1) / 2 - res = torch.exp(-dists) - if diag: - res = torch.diagonal(res, dim1=-1, dim2=-2) - return res diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index 7d860963e..f05baf790 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -7,7 +7,6 @@ import bofire.data_models.kernels.api as data_models import bofire.priors.api as priors -from bofire.kernels.categorical import HammingKernelWithOneHots from bofire.kernels.fingerprint_kernels.tanimoto_kernel import TanimotoKernel from bofire.kernels.shape import WassersteinKernel @@ -216,35 +215,13 @@ def map_HammingDistanceKernel( ard_num_dims: int, active_dims: List[int], features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], -) -> GpytorchKernel: +) -> CategoricalKernel: active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) - - if data_model.with_one_hots is None: - with_one_hots = data_model.features is not None and len(active_dims) > 1 - else: - with_one_hots = data_model.with_one_hots - - if with_one_hots and len(active_dims) == 1: - raise RuntimeError( - "only one feature for categorical kernel operating on one-hot features" - ) - elif not with_one_hots and len(active_dims) > 1: - # this is not necessarily an issue since botorch's CategoricalKernel - # can work on multiple features at the same time - pass - - if with_one_hots: - return HammingKernelWithOneHots( - batch_shape=batch_shape, - ard_num_dims=len(active_dims) if data_model.ard else None, - active_dims=active_dims, # type: ignore - ) - else: - return CategoricalKernel( - batch_shape=batch_shape, - ard_num_dims=len(active_dims) if data_model.ard else None, - active_dims=active_dims, # type: ignore - ) + return CategoricalKernel( + batch_shape=batch_shape, + ard_num_dims=len(active_dims) if data_model.ard else None, + active_dims=active_dims, # type: ignore + ) def map_WassersteinKernel( diff --git a/scratch.py b/scratch.py deleted file mode 100644 index 15caab666..000000000 --- a/scratch.py +++ /dev/null @@ -1,132 +0,0 @@ -import pandas as pd - -import bofire.strategies.api as strategies -import bofire.surrogates.api as surrogates -from bofire.data_models.domain import api as domain_api -from bofire.data_models.features import api as features_api -from bofire.data_models.kernels import api as kernels_api -from bofire.data_models.molfeatures import api as molfeatures_api -from bofire.data_models.priors.api import HVARFNER_LENGTHSCALE_PRIOR -from bofire.data_models.strategies import api as strategies_api -from bofire.data_models.surrogates import api as surrogates_api - - -def test_SingleTaskGPModel_mixed_features(): - """test that we can use a single task gp with mixed features""" - inputs = domain_api.Inputs( - features=[ - features_api.ContinuousInput( - key=f"x_{i+1}", - bounds=(-4, 4), - ) - for i in range(2) - ] - + [ - features_api.CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), - features_api.CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), - ] - ) - outputs = domain_api.Outputs(features=[features_api.ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10) - experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) - experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 - experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 - experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 - experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 - experiments["valid_y"] = 1 - - gp_data = surrogates_api.SingleTaskGPSurrogate( - inputs=inputs, - outputs=outputs, - kernel=kernels_api.AdditiveKernel( - kernels=[ - kernels_api.HammingDistanceKernel( - ard=True, - features=["x_cat_1", "x_cat_2"], - ), - kernels_api.RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=[f"x_{i+1}" for i in range(2)], - ), - ] - ), - ) - - gp_mapped = surrogates.map(gp_data) - assert hasattr(gp_mapped, "fit") - assert len(gp_mapped.kernel.kernels) == 2 - assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] - assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] - gp_mapped.fit(experiments) - pred = gp_mapped.predict(experiments) - assert pred.shape == (10, 2) - assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] - assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] - - -if __name__ == "__main__": - test_SingleTaskGPModel_mixed_features() - - -import sys - - -sys.exit(0) - - -domain = domain_api.Domain( - inputs=domain_api.Inputs( - features=[ - features_api.ContinuousInput(key="x1", bounds=(-1, 1)), - features_api.ContinuousInput(key="x2", bounds=(-1, 1)), - features_api.CategoricalMolecularInput( - key="mol", categories=["CO", "CCO", "CCCO"] - ), - ] - ), - outputs=domain_api.Outputs(features=[features_api.ContinuousOutput(key="f")]), -) - - -strategy = strategies.map( - strategies_api.SoboStrategy( - domain=domain, - surrogate_specs=surrogates_api.BotorchSurrogates( - surrogates=[ - surrogates_api.SingleTaskGPSurrogate( - inputs=domain.inputs, - outputs=domain.outputs, - input_preprocessing_specs={ - "mol": molfeatures_api.Fingerprints(), - }, - kernel=kernels_api.AdditiveKernel( - kernels=[ - kernels_api.RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=["x1", "x2"], - ), - kernels_api.TanimotoKernel( - features=["mol"], - ), - ] - ), - ) - ] - ), - ) -) - - -strategy.tell( - experiments=pd.DataFrame( - [ - {"x1": 0.2, "x2": 0.4, "mol": "CO", "f": 1.0}, - {"x1": 0.4, "x2": 0.2, "mol": "CCO", "f": 2.0}, - {"x1": 0.6, "x2": 0.6, "mol": "CCCO", "f": 3.0}, - ] - ) -) -candidates = strategy.ask(candidate_count=1) -print(candidates) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index d2dbd2861..759aae261 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -335,61 +335,6 @@ def test_SingleTaskGPModel_feature_subsets(): assert len(gp_mapped.model.covar_module.kernels[1].active_dims) == 4 -def test_SingleTaskGPModel_mixed_features(): - """test that we can use a single task gp with mixed features""" - inputs = Inputs( - features=[ - ContinuousInput( - key=f"x_{i+1}", - bounds=(-4, 4), - ) - for i in range(2) - ] - + [ - CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), - CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), - ], - ) - outputs = Outputs(features=[ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10) - experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) - experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 - experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 - experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 - experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 - experiments["valid_y"] = 1 - - gp_data = SingleTaskGPSurrogate( - inputs=inputs, - outputs=outputs, - kernel=AdditiveKernel( - kernels=[ - HammingDistanceKernel( - ard=True, - features=["x_cat_1", "x_cat_2"], - ), - RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=[f"x_{i+1}" for i in range(2)], - ), - ] - ), - ) - - gp_mapped = surrogates.map(gp_data) - assert hasattr(gp_mapped, "fit") - assert len(gp_mapped.kernel.kernels) == 2 - assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] - assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] - gp_mapped.fit(experiments) - pred = gp_mapped.predict(experiments) - assert pred.shape == (10, 2) - assert ((pred['y_pred'] - experiments['y'])**2).mean() < 0.5 - assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] - assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] - - def test_MixedSingleTaskGPHyperconfig(): inputs = Inputs( features=[ From 7e455b74733c8d3fb159defdbc7c1681eaf527b6 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 15:44:59 +0100 Subject: [PATCH 18/43] Revert "Revert "custom hamming kernel enabling single task gp on categorical features"" This reverts commit 2e2985246cfd35cadeef4af4d836e906d259c0c3. --- bofire/data_models/kernels/categorical.py | 3 +- bofire/kernels/categorical.py | 25 ++++ bofire/kernels/mapper.py | 35 +++++- scratch.py | 132 ++++++++++++++++++++++ tests/bofire/surrogates/test_gps.py | 55 +++++++++ 5 files changed, 243 insertions(+), 7 deletions(-) create mode 100644 bofire/kernels/categorical.py create mode 100644 scratch.py diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index 4fa2e0d72..8d03c429d 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -1,4 +1,4 @@ -from typing import Literal +from typing import Literal, Optional from bofire.data_models.kernels.kernel import ConcreteKernel @@ -10,3 +10,4 @@ class CategoricalKernel(ConcreteKernel): class HammingDistanceKernel(CategoricalKernel): type: Literal["HammingDistanceKernel"] = "HammingDistanceKernel" ard: bool = True + with_one_hots: Optional[bool] = None diff --git a/bofire/kernels/categorical.py b/bofire/kernels/categorical.py new file mode 100644 index 000000000..7e04065de --- /dev/null +++ b/bofire/kernels/categorical.py @@ -0,0 +1,25 @@ +import torch +from gpytorch.kernels.kernel import Kernel +from torch import Tensor + + +class HammingKernelWithOneHots(Kernel): + has_lengthscale = True + + def forward( + self, + x1: Tensor, + x2: Tensor, + diag: bool = False, + last_dim_is_batch: bool = False, + ) -> Tensor: + delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3))**2 + dists = delta / self.lengthscale.unsqueeze(-2) + if last_dim_is_batch: + dists = dists.transpose(-3, -1) + + dists = dists.sum(-1) / 2 + res = torch.exp(-dists) + if diag: + res = torch.diagonal(res, dim1=-1, dim2=-2) + return res diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index f05baf790..7d860963e 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -7,6 +7,7 @@ import bofire.data_models.kernels.api as data_models import bofire.priors.api as priors +from bofire.kernels.categorical import HammingKernelWithOneHots from bofire.kernels.fingerprint_kernels.tanimoto_kernel import TanimotoKernel from bofire.kernels.shape import WassersteinKernel @@ -215,13 +216,35 @@ def map_HammingDistanceKernel( ard_num_dims: int, active_dims: List[int], features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], -) -> CategoricalKernel: +) -> GpytorchKernel: active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) - return CategoricalKernel( - batch_shape=batch_shape, - ard_num_dims=len(active_dims) if data_model.ard else None, - active_dims=active_dims, # type: ignore - ) + + if data_model.with_one_hots is None: + with_one_hots = data_model.features is not None and len(active_dims) > 1 + else: + with_one_hots = data_model.with_one_hots + + if with_one_hots and len(active_dims) == 1: + raise RuntimeError( + "only one feature for categorical kernel operating on one-hot features" + ) + elif not with_one_hots and len(active_dims) > 1: + # this is not necessarily an issue since botorch's CategoricalKernel + # can work on multiple features at the same time + pass + + if with_one_hots: + return HammingKernelWithOneHots( + batch_shape=batch_shape, + ard_num_dims=len(active_dims) if data_model.ard else None, + active_dims=active_dims, # type: ignore + ) + else: + return CategoricalKernel( + batch_shape=batch_shape, + ard_num_dims=len(active_dims) if data_model.ard else None, + active_dims=active_dims, # type: ignore + ) def map_WassersteinKernel( diff --git a/scratch.py b/scratch.py new file mode 100644 index 000000000..15caab666 --- /dev/null +++ b/scratch.py @@ -0,0 +1,132 @@ +import pandas as pd + +import bofire.strategies.api as strategies +import bofire.surrogates.api as surrogates +from bofire.data_models.domain import api as domain_api +from bofire.data_models.features import api as features_api +from bofire.data_models.kernels import api as kernels_api +from bofire.data_models.molfeatures import api as molfeatures_api +from bofire.data_models.priors.api import HVARFNER_LENGTHSCALE_PRIOR +from bofire.data_models.strategies import api as strategies_api +from bofire.data_models.surrogates import api as surrogates_api + + +def test_SingleTaskGPModel_mixed_features(): + """test that we can use a single task gp with mixed features""" + inputs = domain_api.Inputs( + features=[ + features_api.ContinuousInput( + key=f"x_{i+1}", + bounds=(-4, 4), + ) + for i in range(2) + ] + + [ + features_api.CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), + features_api.CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), + ] + ) + outputs = domain_api.Outputs(features=[features_api.ContinuousOutput(key="y")]) + experiments = inputs.sample(n=10) + experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) + experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 + experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 + experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 + experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 + experiments["valid_y"] = 1 + + gp_data = surrogates_api.SingleTaskGPSurrogate( + inputs=inputs, + outputs=outputs, + kernel=kernels_api.AdditiveKernel( + kernels=[ + kernels_api.HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + kernels_api.RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=[f"x_{i+1}" for i in range(2)], + ), + ] + ), + ) + + gp_mapped = surrogates.map(gp_data) + assert hasattr(gp_mapped, "fit") + assert len(gp_mapped.kernel.kernels) == 2 + assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] + assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] + gp_mapped.fit(experiments) + pred = gp_mapped.predict(experiments) + assert pred.shape == (10, 2) + assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] + assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] + + +if __name__ == "__main__": + test_SingleTaskGPModel_mixed_features() + + +import sys + + +sys.exit(0) + + +domain = domain_api.Domain( + inputs=domain_api.Inputs( + features=[ + features_api.ContinuousInput(key="x1", bounds=(-1, 1)), + features_api.ContinuousInput(key="x2", bounds=(-1, 1)), + features_api.CategoricalMolecularInput( + key="mol", categories=["CO", "CCO", "CCCO"] + ), + ] + ), + outputs=domain_api.Outputs(features=[features_api.ContinuousOutput(key="f")]), +) + + +strategy = strategies.map( + strategies_api.SoboStrategy( + domain=domain, + surrogate_specs=surrogates_api.BotorchSurrogates( + surrogates=[ + surrogates_api.SingleTaskGPSurrogate( + inputs=domain.inputs, + outputs=domain.outputs, + input_preprocessing_specs={ + "mol": molfeatures_api.Fingerprints(), + }, + kernel=kernels_api.AdditiveKernel( + kernels=[ + kernels_api.RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=["x1", "x2"], + ), + kernels_api.TanimotoKernel( + features=["mol"], + ), + ] + ), + ) + ] + ), + ) +) + + +strategy.tell( + experiments=pd.DataFrame( + [ + {"x1": 0.2, "x2": 0.4, "mol": "CO", "f": 1.0}, + {"x1": 0.4, "x2": 0.2, "mol": "CCO", "f": 2.0}, + {"x1": 0.6, "x2": 0.6, "mol": "CCCO", "f": 3.0}, + ] + ) +) +candidates = strategy.ask(candidate_count=1) +print(candidates) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index 759aae261..d2dbd2861 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -335,6 +335,61 @@ def test_SingleTaskGPModel_feature_subsets(): assert len(gp_mapped.model.covar_module.kernels[1].active_dims) == 4 +def test_SingleTaskGPModel_mixed_features(): + """test that we can use a single task gp with mixed features""" + inputs = Inputs( + features=[ + ContinuousInput( + key=f"x_{i+1}", + bounds=(-4, 4), + ) + for i in range(2) + ] + + [ + CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), + CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), + ], + ) + outputs = Outputs(features=[ContinuousOutput(key="y")]) + experiments = inputs.sample(n=10) + experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) + experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 + experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 + experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 + experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 + experiments["valid_y"] = 1 + + gp_data = SingleTaskGPSurrogate( + inputs=inputs, + outputs=outputs, + kernel=AdditiveKernel( + kernels=[ + HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=[f"x_{i+1}" for i in range(2)], + ), + ] + ), + ) + + gp_mapped = surrogates.map(gp_data) + assert hasattr(gp_mapped, "fit") + assert len(gp_mapped.kernel.kernels) == 2 + assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] + assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] + gp_mapped.fit(experiments) + pred = gp_mapped.predict(experiments) + assert pred.shape == (10, 2) + assert ((pred['y_pred'] - experiments['y'])**2).mean() < 0.5 + assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] + assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] + + def test_MixedSingleTaskGPHyperconfig(): inputs = Inputs( features=[ From 25f947bb777a9717336d3f29fde20efcf66e2223 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 15:45:50 +0100 Subject: [PATCH 19/43] Revert "Revert "testing equivalence of mixed gp and single gp with custom kernel"" This reverts commit 1cd2776d80a44ff11c0329da1f979c70e0298521. --- tests/bofire/surrogates/test_gps.py | 89 +++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index d2dbd2861..3b911f042 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -28,6 +28,7 @@ AdditiveKernel, HammingDistanceKernel, MaternKernel, + MultiplicativeKernel, RBFKernel, ScaleKernel, ) @@ -617,3 +618,91 @@ def test_MixedSingleTaskGPModel_mordred(kernel, scaler, output_scaler): model2.loads(dump) preds2 = model2.predict(experiments.iloc[:-1]) assert_frame_equal(preds, preds2) + + +def test_SingleTaskGP_with_categoricals_is_equivalent_to_MixedSingleTaskGP(): + inputs = Inputs( + features=[ + ContinuousInput( + key=f"x_{i+1}", + bounds=(-4, 4), + ) + for i in range(2) + ] + + [ + CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), + CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), + ] + ) + outputs = Outputs(features=[ContinuousOutput(key="y")]) + experiments = inputs.sample(n=10, seed=194387) + experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) + experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 + experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 + experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 + experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 + experiments["valid_y"] = 1 + + gp_data = SingleTaskGPSurrogate( + inputs=inputs, + outputs=outputs, + kernel=AdditiveKernel( # use the same kernel as botorch.models.MixedSingleTaskGP + kernels=[ + ScaleKernel( + base_kernel=HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ) + ), + ScaleKernel( + base_kernel=RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=[f"x_{i+1}" for i in range(2)], + ) + ), + ScaleKernel( + base_kernel=MultiplicativeKernel( + kernels=[ + HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + features=[f"x_{i+1}" for i in range(2)], + ), + ] + ) + ), + ] + ), + ) + + gp_mapped = surrogates.map(gp_data) + gp_mapped.fit(experiments) + pred = gp_mapped.predict(experiments) + single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() + assert single_task_gp_mse < 3.5 + + model = MixedSingleTaskGPSurrogate( + inputs=inputs, + outputs=outputs, + input_preprocessing_specs={ + "x_cat_1": CategoricalEncodingEnum.ONE_HOT, + "x_cat_2": CategoricalEncodingEnum.ONE_HOT, + }, + continuous_kernel=RBFKernel( + ard=True, + lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), + ), + categorical_kernel=HammingDistanceKernel(ard=True), + ) + model = surrogates.map(model) + model.fit(experiments) + pred = model.predict(experiments) + mixed_single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() + assert mixed_single_task_gp_mse < 3.5 + + assert abs(single_task_gp_mse - mixed_single_task_gp_mse) < 0.005 From 2c145b62135547b15c22b86a28c4d6e431453580 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 15:49:28 +0100 Subject: [PATCH 20/43] removed test debug and restored to latest implemented features --- .github/workflows/test.yaml | 1 - bofire/kernels/categorical.py | 2 +- bofire/strategies/doe/objective.py | 2 -- tests/bofire/strategies/doe/test_objective.py | 11 +---------- tests/bofire/surrogates/test_gps.py | 4 ++-- 5 files changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a2dcfd627..56e950611 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -40,7 +40,6 @@ jobs: testing: runs-on: ubuntu-latest - continue-on-error: true strategy: matrix: python-version: [ '3.9', '3.11' ] diff --git a/bofire/kernels/categorical.py b/bofire/kernels/categorical.py index 7e04065de..640a62d1e 100644 --- a/bofire/kernels/categorical.py +++ b/bofire/kernels/categorical.py @@ -13,7 +13,7 @@ def forward( diag: bool = False, last_dim_is_batch: bool = False, ) -> Tensor: - delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3))**2 + delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3)) ** 2 dists = delta / self.lengthscale.unsqueeze(-2) if last_dim_is_batch: dists = dists.transpose(-3, -1) diff --git a/bofire/strategies/doe/objective.py b/bofire/strategies/doe/objective.py index 473cb19fa..973a5e7de 100644 --- a/bofire/strategies/doe/objective.py +++ b/bofire/strategies/doe/objective.py @@ -54,9 +54,7 @@ def __init__( # terms for model jacobian self.terms_jacobian_t = [] - print('model', model) for var in self.vars: - print('diff', var, model.differentiate(var, use_sympy=True)) _terms = [ str(term).replace(":", "*") + f" + 0 * {self.vars[0]}" for term in model.differentiate(var, use_sympy=True) diff --git a/tests/bofire/strategies/doe/test_objective.py b/tests/bofire/strategies/doe/test_objective.py index b468a1f1b..688dfa014 100644 --- a/tests/bofire/strategies/doe/test_objective.py +++ b/tests/bofire/strategies/doe/test_objective.py @@ -40,16 +40,7 @@ def test_Objective_model_jacobian_t(): model_jacobian_t = objective._model_jacobian_t B = np.zeros(shape=(3, 6)) - B[:, 1:4] = np.eye(3) - B[:, 4] = np.array([0, 0, 6]) - B[:, 5] = np.array([2, 1, 0]) - - J = model_jacobian_t(x) - print('===== B\n', B) - print('===== J\n', J) - print('===== Jterms\n', objective.terms_jacobian_t) - - assert np.allclose(B, J) + assert np.allclose(B, model_jacobian_t(x)) # fully quadratic model f = Formula("x1 + x2 + x3 + x1:x2 + x1:x3 + x2:x3 + {x1**2} + {x2**2} + {x3**2}") diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index 3b911f042..695cf70a6 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -352,7 +352,7 @@ def test_SingleTaskGPModel_mixed_features(): ], ) outputs = Outputs(features=[ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10) + experiments = inputs.sample(n=10, seed=194387) experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 @@ -386,7 +386,7 @@ def test_SingleTaskGPModel_mixed_features(): gp_mapped.fit(experiments) pred = gp_mapped.predict(experiments) assert pred.shape == (10, 2) - assert ((pred['y_pred'] - experiments['y'])**2).mean() < 0.5 + assert ((pred["y_pred"] - experiments["y"]) ** 2).mean() < 0.5 assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] From 30dd1232ceb59f222ee244bd637b0c37cafb9a67 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 16:04:59 +0100 Subject: [PATCH 21/43] pinning compatible version of formulaic --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 085079c3f..d937485db 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ "numpy", "multiprocess", "plotly", - "formulaic>=1.0.1", + "formulaic>=1.0.1,<1.1", "cloudpickle>=2.0.0", "sympy>=1.12", "cvxpy[CLARABEL]", From b53d3bbdbb71a60ace770a8d5fe78497f1b8dc17 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 16:31:45 +0100 Subject: [PATCH 22/43] pinning compatible version of formulaic --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8f1fe26f8..0d0a89026 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ optimization = [ "numpy", "multiprocess", "plotly", - "formulaic>=1.0.1", + "formulaic>=1.0.1,<1.1", "cloudpickle>=2.0.0", "sympy>=1.12", "cvxpy[CLARABEL]", From 8d47cbd79e03527d5639123b4e92781590d25c3f Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 16:33:31 +0100 Subject: [PATCH 23/43] removed old code --- bofire/data_models/kernels/categorical.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index 8d03c429d..e431943fc 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -10,4 +10,3 @@ class CategoricalKernel(ConcreteKernel): class HammingDistanceKernel(CategoricalKernel): type: Literal["HammingDistanceKernel"] = "HammingDistanceKernel" ard: bool = True - with_one_hots: Optional[bool] = None From ce384280cec1b524fb45f42563c60cfa77b2e147 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 16:35:26 +0100 Subject: [PATCH 24/43] lint --- bofire/data_models/kernels/categorical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index e431943fc..4fa2e0d72 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -1,4 +1,4 @@ -from typing import Literal, Optional +from typing import Literal from bofire.data_models.kernels.kernel import ConcreteKernel From 16bdc1f26a467250ded294d0b745b76b44739a4b Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 16:37:31 +0100 Subject: [PATCH 25/43] removed scratch file --- scratch.py | 132 ----------------------------------------------------- 1 file changed, 132 deletions(-) delete mode 100644 scratch.py diff --git a/scratch.py b/scratch.py deleted file mode 100644 index 15caab666..000000000 --- a/scratch.py +++ /dev/null @@ -1,132 +0,0 @@ -import pandas as pd - -import bofire.strategies.api as strategies -import bofire.surrogates.api as surrogates -from bofire.data_models.domain import api as domain_api -from bofire.data_models.features import api as features_api -from bofire.data_models.kernels import api as kernels_api -from bofire.data_models.molfeatures import api as molfeatures_api -from bofire.data_models.priors.api import HVARFNER_LENGTHSCALE_PRIOR -from bofire.data_models.strategies import api as strategies_api -from bofire.data_models.surrogates import api as surrogates_api - - -def test_SingleTaskGPModel_mixed_features(): - """test that we can use a single task gp with mixed features""" - inputs = domain_api.Inputs( - features=[ - features_api.ContinuousInput( - key=f"x_{i+1}", - bounds=(-4, 4), - ) - for i in range(2) - ] - + [ - features_api.CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), - features_api.CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), - ] - ) - outputs = domain_api.Outputs(features=[features_api.ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10) - experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) - experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 - experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 - experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 - experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 - experiments["valid_y"] = 1 - - gp_data = surrogates_api.SingleTaskGPSurrogate( - inputs=inputs, - outputs=outputs, - kernel=kernels_api.AdditiveKernel( - kernels=[ - kernels_api.HammingDistanceKernel( - ard=True, - features=["x_cat_1", "x_cat_2"], - ), - kernels_api.RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=[f"x_{i+1}" for i in range(2)], - ), - ] - ), - ) - - gp_mapped = surrogates.map(gp_data) - assert hasattr(gp_mapped, "fit") - assert len(gp_mapped.kernel.kernels) == 2 - assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] - assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] - gp_mapped.fit(experiments) - pred = gp_mapped.predict(experiments) - assert pred.shape == (10, 2) - assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] - assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] - - -if __name__ == "__main__": - test_SingleTaskGPModel_mixed_features() - - -import sys - - -sys.exit(0) - - -domain = domain_api.Domain( - inputs=domain_api.Inputs( - features=[ - features_api.ContinuousInput(key="x1", bounds=(-1, 1)), - features_api.ContinuousInput(key="x2", bounds=(-1, 1)), - features_api.CategoricalMolecularInput( - key="mol", categories=["CO", "CCO", "CCCO"] - ), - ] - ), - outputs=domain_api.Outputs(features=[features_api.ContinuousOutput(key="f")]), -) - - -strategy = strategies.map( - strategies_api.SoboStrategy( - domain=domain, - surrogate_specs=surrogates_api.BotorchSurrogates( - surrogates=[ - surrogates_api.SingleTaskGPSurrogate( - inputs=domain.inputs, - outputs=domain.outputs, - input_preprocessing_specs={ - "mol": molfeatures_api.Fingerprints(), - }, - kernel=kernels_api.AdditiveKernel( - kernels=[ - kernels_api.RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=["x1", "x2"], - ), - kernels_api.TanimotoKernel( - features=["mol"], - ), - ] - ), - ) - ] - ), - ) -) - - -strategy.tell( - experiments=pd.DataFrame( - [ - {"x1": 0.2, "x2": 0.4, "mol": "CO", "f": 1.0}, - {"x1": 0.4, "x2": 0.2, "mol": "CCO", "f": 2.0}, - {"x1": 0.6, "x2": 0.6, "mol": "CCCO", "f": 3.0}, - ] - ) -) -candidates = strategy.ask(candidate_count=1) -print(candidates) From e306d162717e87b7bd0b5c5fa9b5e276c1a68283 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 16:40:29 +0100 Subject: [PATCH 26/43] removed old code again --- bofire/kernels/mapper.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index 7d860963e..1425d35f1 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -219,11 +219,7 @@ def map_HammingDistanceKernel( ) -> GpytorchKernel: active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) - if data_model.with_one_hots is None: - with_one_hots = data_model.features is not None and len(active_dims) > 1 - else: - with_one_hots = data_model.with_one_hots - + with_one_hots = data_model.features is not None and len(active_dims) > 1 if with_one_hots and len(active_dims) == 1: raise RuntimeError( "only one feature for categorical kernel operating on one-hot features" From 9d5dfc6704f0073ae8a2cbae26569bfd7afbf28d Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 16:45:59 +0100 Subject: [PATCH 27/43] silencing pyright false positive --- bofire/strategies/predictives/botorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bofire/strategies/predictives/botorch.py b/bofire/strategies/predictives/botorch.py index d819864db..cccbd7984 100644 --- a/bofire/strategies/predictives/botorch.py +++ b/bofire/strategies/predictives/botorch.py @@ -656,7 +656,7 @@ def get_categorical_combinations(self) -> List[Dict[int, float]]: elif isinstance(feature, CategoricalMolecularInput): preproc = self.input_preprocessing_specs[feat] - if not isinstance(preproc, AnyMolFeatures): + if not isinstance(preproc, AnyMolFeatures): # type: ignore raise ValueError( f"preprocessing for {feat} must be of type AnyMolFeatures" ) From 62ba2c296919aae4da428e55e6d6c48ee04e2018 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 16:50:31 +0100 Subject: [PATCH 28/43] compatibility with py39 --- bofire/strategies/predictives/botorch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bofire/strategies/predictives/botorch.py b/bofire/strategies/predictives/botorch.py index cccbd7984..63d56bdb4 100644 --- a/bofire/strategies/predictives/botorch.py +++ b/bofire/strategies/predictives/botorch.py @@ -31,7 +31,7 @@ DiscreteInput, Input, ) -from bofire.data_models.molfeatures.api import AnyMolFeatures +from bofire.data_models.molfeatures.api import MolFeatures from bofire.data_models.strategies.api import BotorchStrategy as DataModel from bofire.data_models.strategies.api import RandomStrategy as RandomStrategyDataModel from bofire.data_models.strategies.api import ( @@ -656,7 +656,7 @@ def get_categorical_combinations(self) -> List[Dict[int, float]]: elif isinstance(feature, CategoricalMolecularInput): preproc = self.input_preprocessing_specs[feat] - if not isinstance(preproc, AnyMolFeatures): # type: ignore + if not isinstance(preproc, MolFeatures): raise ValueError( f"preprocessing for {feat} must be of type AnyMolFeatures" ) From d2c1f5d28f69d350409b8cf7966236f1523059f2 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 17:01:32 +0100 Subject: [PATCH 29/43] pin compatible version of formulaic --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8f1fe26f8..0d0a89026 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ optimization = [ "numpy", "multiprocess", "plotly", - "formulaic>=1.0.1", + "formulaic>=1.0.1,<1.1", "cloudpickle>=2.0.0", "sympy>=1.12", "cvxpy[CLARABEL]", From 966bf8ba1f3e9ecdddcf508f9efabf025ab2ab40 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 17:10:54 +0100 Subject: [PATCH 30/43] restored old code --- tests/bofire/strategies/doe/test_objective.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/bofire/strategies/doe/test_objective.py b/tests/bofire/strategies/doe/test_objective.py index 688dfa014..5ab62731e 100644 --- a/tests/bofire/strategies/doe/test_objective.py +++ b/tests/bofire/strategies/doe/test_objective.py @@ -40,6 +40,9 @@ def test_Objective_model_jacobian_t(): model_jacobian_t = objective._model_jacobian_t B = np.zeros(shape=(3, 6)) + B[:, 1:4] = np.eye(3) + B[:, 4] = np.array([0, 0, 6]) + B[:, 5] = np.array([2, 1, 0]) assert np.allclose(B, model_jacobian_t(x)) # fully quadratic model From 231f9f6f4f0c342f2a85bafb07c69d562aa905f6 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 17:19:02 +0100 Subject: [PATCH 31/43] pinning sklearn --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0d0a89026..660b3da0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ all = [ "cloudpickle>=2.0.0", "sympy>=1.12", "cvxpy[CLARABEL]", - "scikit-learn>=1.0.0", + "scikit-learn>=1.0.0,<1.6", "entmoot>=2.0", "lightgbm==4.0.0", "pyomo==6.7.1", From 6a7c9d70a895aa617da2a6b0693b182e7256d3f3 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 17:31:13 +0100 Subject: [PATCH 32/43] pinning sklearn --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0d0a89026..660b3da0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ all = [ "cloudpickle>=2.0.0", "sympy>=1.12", "cvxpy[CLARABEL]", - "scikit-learn>=1.0.0", + "scikit-learn>=1.0.0,<1.6", "entmoot>=2.0", "lightgbm==4.0.0", "pyomo==6.7.1", From 65765476ebdbaf533fdb519400e71a58662f5191 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Thu, 19 Dec 2024 17:38:38 +0100 Subject: [PATCH 33/43] pinning scikit everywhere --- pyproject.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 660b3da0d..3f63940ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,11 +38,11 @@ optimization = [ "cloudpickle>=2.0.0", "sympy>=1.12", "cvxpy[CLARABEL]", - "scikit-learn>=1.0.0", + "scikit-learn>=1.0.0,<1.6", ] entmoot = ["entmoot>=2.0", "lightgbm==4.0.0", "pyomo==6.7.1", "gurobipy"] xgb = ["xgboost>=1.7.5"] -cheminfo = ["rdkit>=2023.3.2", "scikit-learn>=1.0.0", "mordred"] +cheminfo = ["rdkit>=2023.3.2", "scikit-learn>=1.0.0,<1.6", "mordred"] tests = [ "mopti", "pytest", @@ -74,7 +74,6 @@ all = [ "gurobipy", "xgboost>=1.7.5", "rdkit>=2023.3.2", - "scikit-learn>=1.0.0", "mordred", "mopti", "pytest", From e70cc16708b171b2ad7a69016b3bfb44c694e209 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Fri, 20 Dec 2024 10:49:05 +0100 Subject: [PATCH 34/43] not testing for prediction quality --- tests/bofire/surrogates/test_gps.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index e13d98e40..0fe537746 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -386,7 +386,6 @@ def test_SingleTaskGPModel_mixed_features(): gp_mapped.fit(experiments) pred = gp_mapped.predict(experiments) assert pred.shape == (10, 2) - assert ((pred["y_pred"] - experiments["y"]) ** 2).mean() < 0.5 assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] @@ -684,7 +683,6 @@ def test_SingleTaskGP_with_categoricals_is_equivalent_to_MixedSingleTaskGP(): gp_mapped.fit(experiments) pred = gp_mapped.predict(experiments) single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() - assert single_task_gp_mse < 3.5 model = MixedSingleTaskGPSurrogate( inputs=inputs, @@ -703,6 +701,5 @@ def test_SingleTaskGP_with_categoricals_is_equivalent_to_MixedSingleTaskGP(): model.fit(experiments) pred = model.predict(experiments) mixed_single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() - assert mixed_single_task_gp_mse < 3.5 - assert abs(single_task_gp_mse - mixed_single_task_gp_mse) < 0.005 + assert abs(single_task_gp_mse - mixed_single_task_gp_mse) < 1.0 From 54b3c7fdd2bf53bb76e840ec3d947e36adf80425 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Fri, 20 Dec 2024 11:20:55 +0100 Subject: [PATCH 35/43] matching lengthscale constraints in hamming kernel --- bofire/kernels/mapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index 1425d35f1..d0e6d3d52 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -3,6 +3,7 @@ import gpytorch import torch from botorch.models.kernels.categorical import CategoricalKernel +from gpytorch.constraints import GreaterThan from gpytorch.kernels import Kernel as GpytorchKernel import bofire.data_models.kernels.api as data_models @@ -234,6 +235,7 @@ def map_HammingDistanceKernel( batch_shape=batch_shape, ard_num_dims=len(active_dims) if data_model.ard else None, active_dims=active_dims, # type: ignore + lengthscale_constraint=GreaterThan(1e-06), ) else: return CategoricalKernel( From 9b325360b35710836213955f4e3a49398c682e4f Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Fri, 20 Dec 2024 12:25:17 +0100 Subject: [PATCH 36/43] removed equivalence test --- tests/bofire/surrogates/test_gps.py | 90 +---------------------------- 1 file changed, 3 insertions(+), 87 deletions(-) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index 0fe537746..a3e83606e 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -28,7 +28,6 @@ AdditiveKernel, HammingDistanceKernel, MaternKernel, - MultiplicativeKernel, RBFKernel, ScaleKernel, ) @@ -50,6 +49,7 @@ SingleTaskGPSurrogate, ) from bofire.data_models.surrogates.trainable import metrics2objectives +from bofire.kernels.categorical import HammingKernelWithOneHots RDKIT_AVAILABLE = importlib.util.find_spec("rdkit") is not None @@ -381,11 +381,13 @@ def test_SingleTaskGPModel_mixed_features(): gp_mapped = surrogates.map(gp_data) assert hasattr(gp_mapped, "fit") assert len(gp_mapped.kernel.kernels) == 2 + assert isinstance(gp_mapped.kernel.kernels[0], HammingDistanceKernel) assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] gp_mapped.fit(experiments) pred = gp_mapped.predict(experiments) assert pred.shape == (10, 2) + assert isinstance(gp_mapped.model.covar_module.kernels[0], HammingKernelWithOneHots) assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] @@ -617,89 +619,3 @@ def test_MixedSingleTaskGPModel_mordred(kernel, scaler, output_scaler): model2.loads(dump) preds2 = model2.predict(experiments.iloc[:-1]) assert_frame_equal(preds, preds2) - - -def test_SingleTaskGP_with_categoricals_is_equivalent_to_MixedSingleTaskGP(): - inputs = Inputs( - features=[ - ContinuousInput( - key=f"x_{i+1}", - bounds=(-4, 4), - ) - for i in range(2) - ] - + [ - CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), - CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), - ] - ) - outputs = Outputs(features=[ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10, seed=194387) - experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) - experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 - experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 - experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 - experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 - experiments["valid_y"] = 1 - - gp_data = SingleTaskGPSurrogate( - inputs=inputs, - outputs=outputs, - kernel=AdditiveKernel( # use the same kernel as botorch.models.MixedSingleTaskGP - kernels=[ - ScaleKernel( - base_kernel=HammingDistanceKernel( - ard=True, - features=["x_cat_1", "x_cat_2"], - ) - ), - ScaleKernel( - base_kernel=RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=[f"x_{i+1}" for i in range(2)], - ) - ), - ScaleKernel( - base_kernel=MultiplicativeKernel( - kernels=[ - HammingDistanceKernel( - ard=True, - features=["x_cat_1", "x_cat_2"], - ), - RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=[f"x_{i+1}" for i in range(2)], - ), - ] - ) - ), - ] - ), - ) - - gp_mapped = surrogates.map(gp_data) - gp_mapped.fit(experiments) - pred = gp_mapped.predict(experiments) - single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() - - model = MixedSingleTaskGPSurrogate( - inputs=inputs, - outputs=outputs, - input_preprocessing_specs={ - "x_cat_1": CategoricalEncodingEnum.ONE_HOT, - "x_cat_2": CategoricalEncodingEnum.ONE_HOT, - }, - continuous_kernel=RBFKernel( - ard=True, - lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - ), - categorical_kernel=HammingDistanceKernel(ard=True), - ) - model = surrogates.map(model) - model.fit(experiments) - pred = model.predict(experiments) - mixed_single_task_gp_mse = ((pred["y_pred"] - experiments["y"]) ** 2).mean() - - assert abs(single_task_gp_mse - mixed_single_task_gp_mse) < 1.0 From 831a03eafbf3726223756cfd447fc3ba8c0fb2ad Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Fri, 20 Dec 2024 12:25:27 +0100 Subject: [PATCH 37/43] testing hamming kernel --- tests/bofire/kernels/test_categorical.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tests/bofire/kernels/test_categorical.py diff --git a/tests/bofire/kernels/test_categorical.py b/tests/bofire/kernels/test_categorical.py new file mode 100644 index 000000000..3b7ef5b4f --- /dev/null +++ b/tests/bofire/kernels/test_categorical.py @@ -0,0 +1,18 @@ +import torch +from botorch.models.kernels.categorical import CategoricalKernel +from botorch.models.transforms.input import OneHotToNumeric + +from bofire.kernels.categorical import HammingKernelWithOneHots + + +def test_hamming_with_one_hot(): + k1 = CategoricalKernel() + k2 = HammingKernelWithOneHots() + + xin_oh = torch.eye(3) + xin_cat = OneHotToNumeric(3, categorical_features={0: 3}).transform(xin_oh) + + z1 = k1(xin_cat).to_dense() + z2 = k2(xin_oh).to_dense() + + assert torch.allclose(z1, z2) From 561ac20d0e3fd417b58c8d742690d45cac44541d Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Mon, 13 Jan 2025 16:48:08 +0100 Subject: [PATCH 38/43] added test for mol features in single task gp --- tests/bofire/surrogates/test_gps.py | 42 +++++++++++++---------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index a3e83606e..e26d5d1a8 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -30,6 +30,7 @@ MaternKernel, RBFKernel, ScaleKernel, + TanimotoKernel, ) from bofire.data_models.molfeatures.api import MordredDescriptors from bofire.data_models.priors.api import ( @@ -49,7 +50,6 @@ SingleTaskGPSurrogate, ) from bofire.data_models.surrogates.trainable import metrics2objectives -from bofire.kernels.categorical import HammingKernelWithOneHots RDKIT_AVAILABLE = importlib.util.find_spec("rdkit") is not None @@ -340,25 +340,25 @@ def test_SingleTaskGPModel_mixed_features(): """test that we can use a single task gp with mixed features""" inputs = Inputs( features=[ - ContinuousInput( - key=f"x_{i+1}", - bounds=(-4, 4), - ) - for i in range(2) - ] - + [ + ContinuousInput(key="x_1", bounds=(-4, 4)), + ContinuousInput(key="x_2", bounds=(-4, 4)), CategoricalInput(key="x_cat_1", categories=["mama", "papa"]), CategoricalInput(key="x_cat_2", categories=["cat", "dog"]), + MolecularInput(key="x_mol"), ], ) outputs = Outputs(features=[ContinuousOutput(key="y")]) - experiments = inputs.sample(n=10, seed=194387) - experiments.eval("y=((x_1**2 + x_2 - 11)**2+(x_1 + x_2**2 -7)**2)", inplace=True) - experiments.loc[experiments.x_cat_1 == "mama", "y"] *= 5.0 - experiments.loc[experiments.x_cat_1 == "papa", "y"] /= 2.0 - experiments.loc[experiments.x_cat_2 == "cat", "y"] *= -2.0 - experiments.loc[experiments.x_cat_2 == "dog", "y"] /= -5.0 - experiments["valid_y"] = 1 + + experiment_values = [ + [2.56, -1.42, "papa", "dog", -3.98, 1, "CC(=O)Oc1ccccc1C(=O)O"], + [3.84, -2.73, "mama", "cat", -197.46, 1, "c1ccccc1"], + [3.57, 3.23, "papa", "cat", -74.55, 1, "[CH3][CH2][OH]"], + [-0.07, -1.55, "mama", "dog", -179.14, 1, "N[C@](C)(F)C(=O)O"], + ] + experiments = pd.DataFrame( + experiment_values, + columns=["x_1", "x_2", "x_cat_1", "x_cat_2", "y", "valid_y", "x_mol"], + ) gp_data = SingleTaskGPSurrogate( inputs=inputs, @@ -374,22 +374,16 @@ def test_SingleTaskGPModel_mixed_features(): lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), features=[f"x_{i+1}" for i in range(2)], ), + TanimotoKernel(features=["x_mol"]), ] ), ) gp_mapped = surrogates.map(gp_data) - assert hasattr(gp_mapped, "fit") - assert len(gp_mapped.kernel.kernels) == 2 - assert isinstance(gp_mapped.kernel.kernels[0], HammingDistanceKernel) - assert gp_mapped.kernel.kernels[0].features == ["x_cat_1", "x_cat_2"] - assert gp_mapped.kernel.kernels[1].features == ["x_1", "x_2"] gp_mapped.fit(experiments) pred = gp_mapped.predict(experiments) - assert pred.shape == (10, 2) - assert isinstance(gp_mapped.model.covar_module.kernels[0], HammingKernelWithOneHots) - assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [2, 3, 4, 5] - assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] + assert pred.shape == (4, 2) + # assert (pred['y_pred'] - experiments['y']).abs().mean() < 0.4 def test_MixedSingleTaskGPHyperconfig(): From 1867e7b1057ba54d1d5af657ed2e97d3e8e83920 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Mon, 13 Jan 2025 16:48:45 +0100 Subject: [PATCH 39/43] categorical onehot kernel uses the right lengthscale for multiple features --- bofire/kernels/categorical.py | 49 ++++++- bofire/kernels/mapper.py | 37 ++++-- tests/bofire/kernels/test_categorical.py | 68 +++++++++- tests/bofire/kernels/test_mapper.py | 160 +++++++++++++++++++++++ 4 files changed, 295 insertions(+), 19 deletions(-) diff --git a/bofire/kernels/categorical.py b/bofire/kernels/categorical.py index 640a62d1e..8659db6fd 100644 --- a/bofire/kernels/categorical.py +++ b/bofire/kernels/categorical.py @@ -1,24 +1,65 @@ +from typing import Dict + import torch +from botorch.models.transforms.input import OneHotToNumeric from gpytorch.kernels.kernel import Kernel from torch import Tensor class HammingKernelWithOneHots(Kernel): + r""" + A Kernel for one-hot enocded categorical features. The inputs + may contain more than one categorical feature. + + Computes `exp(-dist(x1, x2) / lengthscale)`, where + `dist(x1, x2)` is zero if `x1` and `x2` correspond to the + same category, and one otherwise. If the last dimension + is not a batch dimension, then the mean is considered. + + Note: This kernel is NOT differentiable w.r.t. the inputs. + """ + has_lengthscale = True + def __init__(self, categorical_features: Dict[int, int], *args, **kwargs): + """ + Initialize. + + Args: + categorical_features: A dictionary mapping the starting index of each + categorical feature to its cardinality. This assumes that categoricals + are one-hot encoded. + *args, **kwargs: Passed to gpytorch.kernels.kernel.Kernel + """ + super().__init__(*args, **kwargs) + + onehot_dim = sum(categorical_features.values()) + self.trx = OneHotToNumeric( + onehot_dim, categorical_features=categorical_features + ) + def forward( self, x1: Tensor, x2: Tensor, diag: bool = False, last_dim_is_batch: bool = False, + **params, ) -> Tensor: - delta = (x1.unsqueeze(-2) - x2.unsqueeze(-3)) ** 2 - dists = delta / self.lengthscale.unsqueeze(-2) + x1 = self.trx(x1) + x2 = self.trx(x2) + + delta = x1.unsqueeze(-2) != x2.unsqueeze(-3) + if self.ard_num_dims is not None: + ls = self.lengthscale[..., : delta.shape[-1]] + else: + ls = self.lengthscale + + dists = delta / ls.unsqueeze(-2) if last_dim_is_batch: dists = dists.transpose(-3, -1) - - dists = dists.sum(-1) / 2 + else: + dists = dists.mean(-1) res = torch.exp(-dists) if diag: res = torch.diagonal(res, dim1=-1, dim2=-2) diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index d0e6d3d52..ad8519acd 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -218,22 +218,35 @@ def map_HammingDistanceKernel( active_dims: List[int], features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> GpytorchKernel: - active_dims = _compute_active_dims(data_model, active_dims, features_to_idx_mapper) + if data_model.features is not None: + if features_to_idx_mapper is None: + raise RuntimeError( + "features_to_idx_mapper must be defined when using only a subset of features" + ) - with_one_hots = data_model.features is not None and len(active_dims) > 1 - if with_one_hots and len(active_dims) == 1: - raise RuntimeError( - "only one feature for categorical kernel operating on one-hot features" - ) - elif not with_one_hots and len(active_dims) > 1: - # this is not necessarily an issue since botorch's CategoricalKernel - # can work on multiple features at the same time - pass + active_dims = [] + categorical_features = {} + for k in data_model.features: + idx = features_to_idx_mapper([k]) + categorical_features[len(active_dims)] = len(idx) + + already_used = [i for i in idx if i in active_dims] + if already_used: + raise RuntimeError( + f"indices {already_used} are used in more than one categorical feature" + ) + + active_dims.extend(idx) + + if len(idx) == 1: + raise RuntimeError( + f"feature {k} is supposed to be one-hot encoded but is mapped to a single dimension" + ) - if with_one_hots: return HammingKernelWithOneHots( - batch_shape=batch_shape, + categorical_features=categorical_features, ard_num_dims=len(active_dims) if data_model.ard else None, + batch_shape=batch_shape, active_dims=active_dims, # type: ignore lengthscale_constraint=GreaterThan(1e-06), ) diff --git a/tests/bofire/kernels/test_categorical.py b/tests/bofire/kernels/test_categorical.py index 3b7ef5b4f..20a81d9fe 100644 --- a/tests/bofire/kernels/test_categorical.py +++ b/tests/bofire/kernels/test_categorical.py @@ -5,14 +5,76 @@ from bofire.kernels.categorical import HammingKernelWithOneHots -def test_hamming_with_one_hot(): +def test_hamming_with_one_hot_one_feature(): + cat = {0: 3} + k1 = CategoricalKernel() - k2 = HammingKernelWithOneHots() + k2 = HammingKernelWithOneHots(categorical_features=cat) xin_oh = torch.eye(3) - xin_cat = OneHotToNumeric(3, categorical_features={0: 3}).transform(xin_oh) + xin_cat = OneHotToNumeric(3, categorical_features=cat).transform(xin_oh) z1 = k1(xin_cat).to_dense() z2 = k2(xin_oh).to_dense() + assert z1.shape == z2.shape == (3, 3) + assert torch.allclose(z1, z2) + + +def test_hamming_with_one_hot_two_features(): + cat = {0: 2, 2: 4} + + k1 = CategoricalKernel() + k2 = HammingKernelWithOneHots(categorical_features=cat) + + xin_oh = torch.zeros(4, 6) + xin_oh[:2, :2] = xin_oh[2:, :2] = torch.eye(2) + xin_oh[:, 2:] = torch.eye(4) + + xin_cat = OneHotToNumeric(6, categorical_features=cat).transform(xin_oh) + + z1 = k1(xin_cat).to_dense() + z2 = k2(xin_oh).to_dense() + + assert z1.shape == z2.shape == (4, 4) + assert torch.allclose(z1, z2) + + +def test_hamming_with_one_hot_two_features_and_lengthscales(): + cat = {0: 2, 2: 4} + + k1 = CategoricalKernel(ard_num_dims=2) + k1.lengthscale = torch.tensor([1.5, 3.0]) + + # botorch will check that the lengthscale for ARD has the same number of elements as the one-hotted inputs, + # so we have to specify the ard_num_dims accordingly. The kernel will make sure to only use the right + # number of elements, corresponding to the number of categorical features. + k2 = HammingKernelWithOneHots(categorical_features=cat, ard_num_dims=6) + k2.lengthscale = torch.tensor([1.5, 3.0, 0.0, 0.0, 0.0, 0.0]) + + xin_oh = torch.zeros(4, 6) + xin_oh[:2, :2] = xin_oh[2:, :2] = torch.eye(2) + xin_oh[:, 2:] = torch.eye(4) + + xin_cat = OneHotToNumeric(6, categorical_features=cat).transform(xin_oh) + + z1 = k1(xin_cat).to_dense() + z2 = k2(xin_oh).to_dense() + + assert z1.shape == z2.shape == (4, 4) + assert torch.allclose(z1, z2) + + +def test_feature_order(): + x1_in = torch.zeros(4, 2) + x1_in[:2, :] = x1_in[2:, :] = torch.eye(2) + x2_in = torch.eye(4) + + k1 = HammingKernelWithOneHots(categorical_features={0: 2, 2: 4}) + k2 = HammingKernelWithOneHots(categorical_features={0: 4, 4: 2}) + + z1 = k1(torch.cat([x1_in, x2_in], dim=1)).to_dense() + z2 = k2(torch.cat([x2_in, x1_in], dim=1)).to_dense() + + assert z1.shape == z2.shape == (4, 4) assert torch.allclose(z1, z2) diff --git a/tests/bofire/kernels/test_mapper.py b/tests/bofire/kernels/test_mapper.py index fec7d8341..73bba9333 100644 --- a/tests/bofire/kernels/test_mapper.py +++ b/tests/bofire/kernels/test_mapper.py @@ -21,6 +21,7 @@ WassersteinKernel, ) from bofire.data_models.priors.api import THREESIX_SCALE_PRIOR, GammaPrior +from bofire.kernels.categorical import HammingKernelWithOneHots from tests.bofire.data_models.specs.api import Spec @@ -250,3 +251,162 @@ def test_map_wasserstein_kernel(): ) assert k.squared is True assert hasattr(k, "lengthscale_prior") is False + + +def test_map_HammingDistanceKernel_to_onehot_with_ard(): + fmap = { + "x_cat_1": [5, 6, 7, 8], + "x_cat_2": [2, 3], + } + + k_mapped = kernels.map( + HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + batch_shape=torch.Size(), + ard_num_dims=10, + active_dims=list(range(5)), + features_to_idx_mapper=lambda ks: [i for k in ks for i in fmap[k]], + ) + + assert isinstance(k_mapped, HammingKernelWithOneHots) + assert k_mapped.active_dims.tolist() == [5, 6, 7, 8, 2, 3] + assert k_mapped.ard_num_dims == 6 + assert k_mapped.lengthscale.shape == (1, 6) + assert k_mapped.trx.categorical_features == {0: 4, 4: 2} + + +def test_map_HammingDistanceKernel_to_onehot_without_ard(): + fmap = { + "x_cat_1": [5, 6, 7, 8], + "x_cat_2": [2, 3], + } + + k_mapped = kernels.map( + HammingDistanceKernel( + ard=False, + features=["x_cat_1", "x_cat_2"], + ), + batch_shape=torch.Size(), + ard_num_dims=10, + active_dims=list(range(5)), + features_to_idx_mapper=lambda ks: [i for k in ks for i in fmap[k]], + ) + + assert isinstance(k_mapped, HammingKernelWithOneHots) + assert k_mapped.active_dims.tolist() == [5, 6, 7, 8, 2, 3] + assert k_mapped.ard_num_dims is None + assert k_mapped.lengthscale.shape == (1, 1) + assert k_mapped.trx.categorical_features == {0: 4, 4: 2} + + +def test_map_HammingDistanceKernel_to_categorical_without_ard(): + k_mapped = kernels.map( + HammingDistanceKernel( + ard=False, + ), + batch_shape=torch.Size(), + ard_num_dims=10, + active_dims=list(range(5)), + features_to_idx_mapper=None, + ) + + assert isinstance(k_mapped, CategoricalKernel) + assert k_mapped.active_dims.tolist() == [0, 1, 2, 3, 4] + assert k_mapped.ard_num_dims is None + assert k_mapped.lengthscale.shape == (1, 1) + + +def test_map_HammingDistanceKernel_to_categorical_with_ard(): + k_mapped = kernels.map( + HammingDistanceKernel( + ard=True, + ), + batch_shape=torch.Size(), + ard_num_dims=10, + active_dims=list(range(5)), + features_to_idx_mapper=None, + ) + + assert isinstance(k_mapped, CategoricalKernel) + assert k_mapped.active_dims.tolist() == [0, 1, 2, 3, 4] + assert k_mapped.ard_num_dims == 5 + assert k_mapped.lengthscale.shape == (1, 5) + + +def test_map_HammingDistanceKernel_to_onehot_checks_dimension_overlap(): + fmap = { + "x_cat_1": [3, 4], + "x_cat_2": [2, 3], + } + + with pytest.raises(RuntimeError): + kernels.map( + HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + batch_shape=torch.Size(), + ard_num_dims=10, + active_dims=list(range(5)), + features_to_idx_mapper=lambda ks: [i for k in ks for i in fmap[k]], + ) + + +def test_map_HammingDistanceKernel_to_onehot_checks_onehot_encoding(): + fmap = { + "x_cat_1": [4], + "x_cat_2": [2, 3], + } + + with pytest.raises(RuntimeError): + kernels.map( + HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + batch_shape=torch.Size(), + ard_num_dims=10, + active_dims=list(range(5)), + features_to_idx_mapper=lambda ks: [i for k in ks for i in fmap[k]], + ) + + +def test_map_multiple_kernels_on_feature_subsets(): + fmap = { + "x_1": [0], + "x_2": [1], + "x_cat_1": [2, 3], + "x_cat_2": [4, 5], + } + + k_mapped = kernels.map( + AdditiveKernel( + kernels=[ + HammingDistanceKernel( + ard=True, + features=["x_cat_1", "x_cat_2"], + ), + RBFKernel( + features=["x_1", "x_2"], + ), + ] + ), + batch_shape=torch.Size(), + ard_num_dims=10, + active_dims=list(range(5)), + features_to_idx_mapper=lambda ks: [i for k in ks for i in fmap[k]], + ) + + assert len(k_mapped.kernels) == 2 + + assert isinstance(k_mapped.kernels[0], HammingKernelWithOneHots) + assert k_mapped.kernels[0].active_dims.tolist() == [2, 3, 4, 5] + assert k_mapped.kernels[0].ard_num_dims == 4 + + from gpytorch.kernels import RBFKernel as GpytorchRBFKernel + + assert isinstance(k_mapped.kernels[1], GpytorchRBFKernel) + assert k_mapped.kernels[1].active_dims.tolist() == [0, 1] + assert k_mapped.kernels[1].ard_num_dims == 2 From f30ed6ddf3c74f832fc66c826caa291a022f1d1d Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Mon, 13 Jan 2025 16:49:27 +0100 Subject: [PATCH 40/43] removed redundant check --- bofire/data_models/domain/features.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/bofire/data_models/domain/features.py b/bofire/data_models/domain/features.py index 57b01b2d0..ea2975846 100644 --- a/bofire/data_models/domain/features.py +++ b/bofire/data_models/domain/features.py @@ -593,7 +593,6 @@ def _validate_transform_specs( """ # first check that the keys in the specs dict are correct also correct feature keys # next check that all values are of type CategoricalEncodingEnum or MolFeatures - checked_keys = set() for key, value in specs.items(): try: feat = self.get_by_key(key) @@ -623,20 +622,6 @@ def _validate_transform_specs( raise ValueError( f"Forbidden transform type for feature with key {key}", ) - checked_keys.add(key) - - # now check that features that must be transformed do have a transformation defined - for key in self.get_keys(): - if key in checked_keys: - continue - - feat = self.get_by_key(key) - if isinstance(feat, MolecularInput): - trx = specs.get(key) - if trx is None or not isinstance(trx, MolFeatures): - raise ValueError( - "MolecularInput features must have a input processing of type MolFeatures defined" - ) return specs From 7afcd7ccfc9f57509867ab81176061a6f6bcf8d4 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Mon, 13 Jan 2025 16:52:25 +0100 Subject: [PATCH 41/43] more descriptive name for base kernel --- bofire/data_models/kernels/api.py | 8 ++++++-- bofire/data_models/kernels/categorical.py | 4 ++-- bofire/data_models/kernels/continuous.py | 4 ++-- bofire/data_models/kernels/kernel.py | 2 +- bofire/data_models/kernels/molecular.py | 4 ++-- bofire/kernels/mapper.py | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/bofire/data_models/kernels/api.py b/bofire/data_models/kernels/api.py index ce37ad223..0fffbaa5f 100644 --- a/bofire/data_models/kernels/api.py +++ b/bofire/data_models/kernels/api.py @@ -17,7 +17,11 @@ PolynomialKernel, RBFKernel, ) -from bofire.data_models.kernels.kernel import AggregationKernel, ConcreteKernel, Kernel +from bofire.data_models.kernels.kernel import ( + AggregationKernel, + FeatureSpecificKernel, + Kernel, +) from bofire.data_models.kernels.molecular import MolecularKernel, TanimotoKernel from bofire.data_models.kernels.shape import WassersteinKernel @@ -27,7 +31,7 @@ CategoricalKernel, ContinuousKernel, MolecularKernel, - ConcreteKernel, + FeatureSpecificKernel, AggregationKernel, ] diff --git a/bofire/data_models/kernels/categorical.py b/bofire/data_models/kernels/categorical.py index 4fa2e0d72..3081c7148 100644 --- a/bofire/data_models/kernels/categorical.py +++ b/bofire/data_models/kernels/categorical.py @@ -1,9 +1,9 @@ from typing import Literal -from bofire.data_models.kernels.kernel import ConcreteKernel +from bofire.data_models.kernels.kernel import FeatureSpecificKernel -class CategoricalKernel(ConcreteKernel): +class CategoricalKernel(FeatureSpecificKernel): pass diff --git a/bofire/data_models/kernels/continuous.py b/bofire/data_models/kernels/continuous.py index 3516648e7..bf2983159 100644 --- a/bofire/data_models/kernels/continuous.py +++ b/bofire/data_models/kernels/continuous.py @@ -2,11 +2,11 @@ from pydantic import PositiveInt, field_validator -from bofire.data_models.kernels.kernel import ConcreteKernel +from bofire.data_models.kernels.kernel import FeatureSpecificKernel from bofire.data_models.priors.api import AnyGeneralPrior, AnyPrior -class ContinuousKernel(ConcreteKernel): +class ContinuousKernel(FeatureSpecificKernel): pass diff --git a/bofire/data_models/kernels/kernel.py b/bofire/data_models/kernels/kernel.py index c31744aff..5673baf18 100644 --- a/bofire/data_models/kernels/kernel.py +++ b/bofire/data_models/kernels/kernel.py @@ -11,5 +11,5 @@ class AggregationKernel(Kernel): pass -class ConcreteKernel(Kernel): +class FeatureSpecificKernel(Kernel): features: Optional[List[str]] = None diff --git a/bofire/data_models/kernels/molecular.py b/bofire/data_models/kernels/molecular.py index c30932a40..b4ead51d4 100644 --- a/bofire/data_models/kernels/molecular.py +++ b/bofire/data_models/kernels/molecular.py @@ -1,9 +1,9 @@ from typing import Literal -from bofire.data_models.kernels.kernel import ConcreteKernel +from bofire.data_models.kernels.kernel import FeatureSpecificKernel -class MolecularKernel(ConcreteKernel): +class MolecularKernel(FeatureSpecificKernel): pass diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index ad8519acd..a68d975b3 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -14,7 +14,7 @@ def _compute_active_dims( - data_model: data_models.ConcreteKernel, + data_model: data_models.FeatureSpecificKernel, active_dims: List[int], features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> List[int]: From d6e2957433784458c83dfbb5685c27c394457eaf Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Mon, 13 Jan 2025 16:57:51 +0100 Subject: [PATCH 42/43] updated docstring --- bofire/kernels/categorical.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bofire/kernels/categorical.py b/bofire/kernels/categorical.py index 8659db6fd..f966bd895 100644 --- a/bofire/kernels/categorical.py +++ b/bofire/kernels/categorical.py @@ -9,8 +9,10 @@ class HammingKernelWithOneHots(Kernel): r""" A Kernel for one-hot enocded categorical features. The inputs - may contain more than one categorical feature. + may contain more than one categorical feature. + This kernel mimics the functionality of CategoricalKernel from + botorch, but assumes categorical features encoded as one-hot variables. Computes `exp(-dist(x1, x2) / lengthscale)`, where `dist(x1, x2)` is zero if `x1` and `x2` correspond to the same category, and one otherwise. If the last dimension @@ -29,7 +31,7 @@ def __init__(self, categorical_features: Dict[int, int], *args, **kwargs): categorical_features: A dictionary mapping the starting index of each categorical feature to its cardinality. This assumes that categoricals are one-hot encoded. - *args, **kwargs: Passed to gpytorch.kernels.kernel.Kernel + *args, **kwargs: Passed to gpytorch.kernels.kernel.Kernel.__init__ """ super().__init__(*args, **kwargs) From 16d831cdee767e8338c7c22412bbda3c5fae3689 Mon Sep 17 00:00:00 2001 From: e-dorigatti Date: Tue, 14 Jan 2025 20:35:18 +0100 Subject: [PATCH 43/43] improved tests and comments --- bofire/kernels/categorical.py | 4 +- bofire/kernels/mapper.py | 8 +++- tests/bofire/data_models/specs/kernels.py | 7 ++++ tests/bofire/kernels/test_mapper.py | 45 ++++++++++++++++++++++- tests/bofire/surrogates/test_gps.py | 16 ++++++-- 5 files changed, 73 insertions(+), 7 deletions(-) diff --git a/bofire/kernels/categorical.py b/bofire/kernels/categorical.py index f966bd895..95564d864 100644 --- a/bofire/kernels/categorical.py +++ b/bofire/kernels/categorical.py @@ -9,7 +9,7 @@ class HammingKernelWithOneHots(Kernel): r""" A Kernel for one-hot enocded categorical features. The inputs - may contain more than one categorical feature. + may contain more than one categorical feature. This kernel mimics the functionality of CategoricalKernel from botorch, but assumes categorical features encoded as one-hot variables. @@ -53,6 +53,8 @@ def forward( delta = x1.unsqueeze(-2) != x2.unsqueeze(-3) if self.ard_num_dims is not None: + # botorch forces ard_num_dims to be the same as the total size of the of one-hot encoded features + # however here we just need one length scale per categorical feature ls = self.lengthscale[..., : delta.shape[-1]] else: ls = self.lengthscale diff --git a/bofire/kernels/mapper.py b/bofire/kernels/mapper.py index a68d975b3..3cedc0812 100644 --- a/bofire/kernels/mapper.py +++ b/bofire/kernels/mapper.py @@ -19,7 +19,10 @@ def _compute_active_dims( features_to_idx_mapper: Optional[Callable[[List[str]], List[int]]], ) -> List[int]: if data_model.features: - assert features_to_idx_mapper is not None + if features_to_idx_mapper is None: + raise RuntimeError( + "features_to_idx_mapper must be defined when using only a subset of features" + ) active_dims = features_to_idx_mapper(data_model.features) return active_dims @@ -245,6 +248,9 @@ def map_HammingDistanceKernel( return HammingKernelWithOneHots( categorical_features=categorical_features, + # botorch will check that the lengthscale for ARD has the same number of elements as the one-hotted inputs, + # so we have to specify the ard_num_dims accordingly. The kernel will make sure to only use one length scale + # for each categorical feature. ard_num_dims=len(active_dims) if data_model.ard else None, batch_shape=batch_shape, active_dims=active_dims, # type: ignore diff --git a/tests/bofire/data_models/specs/kernels.py b/tests/bofire/data_models/specs/kernels.py index 96475a811..c75e4ed21 100644 --- a/tests/bofire/data_models/specs/kernels.py +++ b/tests/bofire/data_models/specs/kernels.py @@ -13,6 +13,13 @@ "features": None, }, ) +specs.add_valid( + kernels.HammingDistanceKernel, + lambda: { + "ard": True, + "features": ["x_cat_1", "x_cat_2"], + }, +) specs.add_valid( kernels.WassersteinKernel, lambda: { diff --git a/tests/bofire/kernels/test_mapper.py b/tests/bofire/kernels/test_mapper.py index 73bba9333..d99604554 100644 --- a/tests/bofire/kernels/test_mapper.py +++ b/tests/bofire/kernels/test_mapper.py @@ -9,6 +9,7 @@ import bofire.kernels.shape as shapeKernels from bofire.data_models.kernels.api import ( AdditiveKernel, + FeatureSpecificKernel, HammingDistanceKernel, InfiniteWidthBNNKernel, LinearKernel, @@ -22,6 +23,7 @@ ) from bofire.data_models.priors.api import THREESIX_SCALE_PRIOR, GammaPrior from bofire.kernels.categorical import HammingKernelWithOneHots +from bofire.kernels.mapper import _compute_active_dims from tests.bofire.data_models.specs.api import Spec @@ -341,7 +343,10 @@ def test_map_HammingDistanceKernel_to_onehot_checks_dimension_overlap(): "x_cat_2": [2, 3], } - with pytest.raises(RuntimeError): + with pytest.raises( + RuntimeError, + match=r"indices \[3\] are used in more than one categorical feature", + ): kernels.map( HammingDistanceKernel( ard=True, @@ -360,7 +365,10 @@ def test_map_HammingDistanceKernel_to_onehot_checks_onehot_encoding(): "x_cat_2": [2, 3], } - with pytest.raises(RuntimeError): + with pytest.raises( + RuntimeError, + match="feature x_cat_1 is supposed to be one-hot encoded but is mapped to a single dimension", + ): kernels.map( HammingDistanceKernel( ard=True, @@ -410,3 +418,36 @@ def test_map_multiple_kernels_on_feature_subsets(): assert isinstance(k_mapped.kernels[1], GpytorchRBFKernel) assert k_mapped.kernels[1].active_dims.tolist() == [0, 1] assert k_mapped.kernels[1].ard_num_dims == 2 + + +def test_compute_active_dims_no_features_returns_active_dims(): + assert _compute_active_dims( + data_model=FeatureSpecificKernel( + type="test", + features=None, + ), + active_dims=[1, 2, 3], + features_to_idx_mapper=None, + ) == [1, 2, 3] + + +def test_compute_active_dims_features_override_active_dims(): + assert _compute_active_dims( + data_model=FeatureSpecificKernel(type="test", features=["x1", "x2"]), + active_dims=[1, 2, 3], + features_to_idx_mapper=lambda ks: [ + i for k in ks for i in {"x1": [4], "x2": [7]}[k] + ], + ) == [4, 7] + + +def test_compute_active_dims_fails_with_features_without_mapper(): + with pytest.raises( + RuntimeError, + match="features_to_idx_mapper must be defined when using only a subset of features", + ): + _compute_active_dims( + data_model=FeatureSpecificKernel(type="test", features=["x1", "x2"]), + active_dims=[1, 2, 3], + features_to_idx_mapper=None, + ) diff --git a/tests/bofire/surrogates/test_gps.py b/tests/bofire/surrogates/test_gps.py index e26d5d1a8..443f51c51 100644 --- a/tests/bofire/surrogates/test_gps.py +++ b/tests/bofire/surrogates/test_gps.py @@ -332,8 +332,8 @@ def test_SingleTaskGPModel_feature_subsets(): gp_mapped.fit(bench_expts) pred = gp_mapped.predict(bench_expts) assert pred.shape == (12, 2) - assert len(gp_mapped.model.covar_module.kernels[0].active_dims) == 2 - assert len(gp_mapped.model.covar_module.kernels[1].active_dims) == 4 + assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [0, 1] + assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [2, 3, 4, 5] def test_SingleTaskGPModel_mixed_features(): @@ -372,7 +372,7 @@ def test_SingleTaskGPModel_mixed_features(): RBFKernel( ard=True, lengthscale_prior=HVARFNER_LENGTHSCALE_PRIOR(), - features=[f"x_{i+1}" for i in range(2)], + features=["x_1", "x_2"], ), TanimotoKernel(features=["x_mol"]), ] @@ -383,6 +383,16 @@ def test_SingleTaskGPModel_mixed_features(): gp_mapped.fit(experiments) pred = gp_mapped.predict(experiments) assert pred.shape == (4, 2) + assert gp_mapped.model.covar_module.kernels[0].active_dims.tolist() == [ + 2050, + 2051, + 2052, + 2053, + ] + assert gp_mapped.model.covar_module.kernels[1].active_dims.tolist() == [0, 1] + assert gp_mapped.model.covar_module.kernels[2].active_dims.tolist() == list( + range(2, 2050) + ) # assert (pred['y_pred'] - experiments['y']).abs().mean() < 0.4