Custom objectives

Bayesian Optimization with Custom Objectives¶

In this tutorial we demonstrate the use of Xopt to preform Bayesian Optimization on custom objectives. In this case, we develop models of individual components of the objective function and combine samples from these models to calculate predicted objective values.

In this example we try to maximize the objective function $$f(g_1(x),g_2(x)) = \min(g_1(x), g_2(x))$$ where $g_1(x) = (x-0.5)^2$ and $g_2(x) = (x - 2)^2$.

Define the test problem¶

In [1]:

Copied!





from xopt.vocs import VOCS

import torch

from xopt.evaluator import Evaluator
from xopt.generators.bayesian import ExpectedImprovementGenerator
from xopt import Xopt
from xopt.generators.bayesian.objectives import CustomXoptObjective
from torch import Tensor
from typing import Optional


# define variables and function objectives
vocs = VOCS(variables={"x": [0.0, 2.0]}, observables=["g1", "g2"])
from xopt.vocs import VOCS

import torch

from xopt.evaluator import Evaluator
from xopt.generators.bayesian import ExpectedImprovementGenerator
from xopt import Xopt
from xopt.generators.bayesian.objectives import CustomXoptObjective
from torch import Tensor
from typing import Optional


# define variables and function objectives
vocs = VOCS(variables={"x": [0.0, 2.0]}, observables=["g1", "g2"])

In [2]:

Copied!

# define a test function to optimize

def sin_function(input_dict):
    return {"g1": (input_dict["x"]) ** 2, "g2": (input_dict["x"] - 2.0) ** 2}
# define a test function to optimize

def sin_function(input_dict):
    return {"g1": (input_dict["x"]) ** 2, "g2": (input_dict["x"] - 2.0) ** 2}

Create Xopt objects¶

Create the evaluator to evaluate our test function and create a generator that uses the Upper Confidence Bound acquisition function to perform Bayesian Optimization.

In [3]:

Copied!





class MyObjective(CustomXoptObjective):
    def forward(self, samples: Tensor, X: Optional[Tensor] = None) -> Tensor:
        return torch.min(
            samples[..., self.vocs.output_names.index("g1")],
            samples[..., self.vocs.output_names.index("g2")],
        )


evaluator = Evaluator(function=sin_function)
generator = ExpectedImprovementGenerator(
    vocs=vocs,
    custom_objective=MyObjective(vocs),
    log_transform_acquisition_function=True,
)
X = Xopt(evaluator=evaluator, generator=generator, vocs=vocs)
print(X)

X.random_evaluate(2)

n_steps = 5

# test points for plotting
test_x = torch.linspace(*X.vocs.bounds.flatten(), 50).double()

for i in range(n_steps):
    # get the Gaussian process model from the generator
    model = X.generator.train_model()

    X.generator.visualize_model()

    # do the optimization step
    X.step()
class MyObjective(CustomXoptObjective):
    def forward(self, samples: Tensor, X: Optional[Tensor] = None) -> Tensor:
        return torch.min(
            samples[..., self.vocs.output_names.index("g1")],
            samples[..., self.vocs.output_names.index("g2")],
        )


evaluator = Evaluator(function=sin_function)
generator = ExpectedImprovementGenerator(
    vocs=vocs,
    custom_objective=MyObjective(vocs),
    log_transform_acquisition_function=True,
)
X = Xopt(evaluator=evaluator, generator=generator, vocs=vocs)
print(X)

X.random_evaluate(2)

n_steps = 5

# test points for plotting
test_x = torch.linspace(*X.vocs.bounds.flatten(), 50).double()

for i in range(n_steps):
    # get the Gaussian process model from the generator
    model = X.generator.train_model()

    X.generator.visualize_model()

    # do the optimization step
    X.step()

            Xopt
________________________________
Version: 2.4.6.dev5+ga295b108.d20250107
Data size: 0
Config as YAML:
dump_file: null
evaluator:
  function: __main__.sin_function
  function_kwargs: {}
  max_workers: 1
  vectorized: false
generator:
  computation_time: null
  fixed_features: null
  gp_constructor:
    covar_modules: {}
    custom_noise_prior: null
    mean_modules: {}
    name: standard
    trainable_mean_keys: []
    transform_inputs: true
    use_cached_hyperparameters: false
    use_low_noise_prior: true
  log_transform_acquisition_function: true
  max_travel_distances: null
  memory_length: null
  model: null
  n_candidates: 1
  n_interpolate_points: null
  n_monte_carlo_samples: 128
  name: expected_improvement
  numerical_optimizer:
    max_iter: 2000
    max_time: null
    n_restarts: 20
    name: LBFGS
  supports_batch_generation: true
  turbo_controller: null
  use_cuda: false
max_evaluations: null
serialize_inline: false
serialize_torch: false
strict: true
vocs:
  constants: {}
  constraints: {}
  objectives: {}
  observables:
  - g1
  - g2
  variables:
    x:
    - 0.0
    - 2.0

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

/home/runner/miniconda3/envs/xopt-dev/lib/python3.13/site-packages/botorch/acquisition/monte_carlo.py:393: NumericsWarning: qExpectedImprovement has known numerical issues that lead to suboptimal optimization performance. It is strongly recommended to simply replace

	 qExpectedImprovement 	 --> 	 qLogExpectedImprovement 

instead, which fixes the issues and has the same API. See https://arxiv.org/abs/2310.20708 for details.
  legacy_ei_numerics_warning(legacy_name=type(self).__name__)

No description has been provided for this image

In [ ]:

In [4]:

Copied!

# access the collected data
X.data
# access the collected data
X.data

Out[4]:

	x	g1	g2	xopt_runtime	xopt_error
0	0.180146	0.032452	3.311870	0.000003	False
1	0.878066	0.770999	1.258737	0.000002	False
2	1.186373	1.407481	0.661989	0.000003	False
3	1.929101	3.721430	0.005027	0.000003	False
4	0.999394	0.998788	1.001213	0.000003	False
5	0.998447	0.996897	1.003108	0.000003	False
6	0.382928	0.146634	2.614921	0.000003	False

Getting the optimization result¶

To get the best point (without evaluating it) we ask the generator to predict the optimum based on the posterior mean.

In [5]:

Copied!

X.generator.get_optimum()
X.generator.get_optimum()

Out[5]:

	x
0	0.999987

Customizing optimization¶

Each generator has a set of options that can be modified to effect optimization behavior

In [6]:

Copied!

X.generator.dict()
X.generator.dict()

/tmp/ipykernel_4378/1542263183.py:1: PydanticDeprecatedSince20: The `dict` method is deprecated; use `model_dump` instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.10/migration/
  X.generator.dict()

Out[6]:

{'model': ModelListGP(
   (models): ModuleList(
     (0-1): 2 x SingleTaskGP(
       (likelihood): GaussianLikelihood(
         (noise_covar): HomoskedasticNoise(
           (noise_prior): GammaPrior()
           (raw_noise_constraint): GreaterThan(1.000E-04)
         )
       )
       (mean_module): ConstantMean()
       (covar_module): RBFKernel(
         (lengthscale_prior): LogNormalPrior()
         (raw_lengthscale_constraint): GreaterThan(2.500E-02)
       )
       (outcome_transform): Standardize()
       (input_transform): Normalize()
     )
   )
   (likelihood): LikelihoodList(
     (likelihoods): ModuleList(
       (0-1): 2 x GaussianLikelihood(
         (noise_covar): HomoskedasticNoise(
           (noise_prior): GammaPrior()
           (raw_noise_constraint): GreaterThan(1.000E-04)
         )
       )
     )
   )
 ),
 'n_monte_carlo_samples': 128,
 'turbo_controller': None,
 'use_cuda': False,
 'gp_constructor': {'name': 'standard',
  'use_low_noise_prior': True,
  'covar_modules': {},
  'mean_modules': {},
  'trainable_mean_keys': [],
  'transform_inputs': True,
  'custom_noise_prior': None,
  'use_cached_hyperparameters': False},
 'numerical_optimizer': {'name': 'LBFGS',
  'n_restarts': 20,
  'max_iter': 2000,
  'max_time': None},
 'max_travel_distances': None,
 'fixed_features': None,
 'computation_time':    training  acquisition_optimization
 0  0.146190                  0.382422
 1  0.146766                  0.591754
 2  0.137059                  0.177567
 3  0.161265                  0.363072
 4  0.137193                  0.020389,
 'log_transform_acquisition_function': True,
 'custom_objective': MyObjective(),
 'n_interpolate_points': None,
 'memory_length': None,
 'n_candidates': 1}