Basic example

Building GP Models from Scratch¶

Sometimes it is useful to build GP models outside the context of BO for data visualization and senativity measurements, ie. learned hyperparameters. Here we demonstrate how to build models from data outside of generators.

For this we use the 3D rosenbrock function test function.

In [1]:

# set values if testing
import os

from xopt import Xopt, Evaluator
from xopt.generators import RandomGenerator
from xopt.resources.test_functions.rosenbrock import (
    evaluate_rosenbrock,
    make_rosenbrock_vocs,
)

from xopt.generators.bayesian.visualize import visualize_model
from xopt.generators.bayesian.models.standard import StandardModelConstructor

# Ignore all warnings
import warnings

warnings.filterwarnings("ignore")


SMOKE_TEST = os.environ.get("SMOKE_TEST")
NUM_MC_SAMPLES = 1 if SMOKE_TEST else 128
NUM_RESTARTS = 1 if SMOKE_TEST else 20

# make rosenbrock function vocs in 3D
vocs = make_rosenbrock_vocs(3)

# collect some data using random sampling
evaluator = Evaluator(function=evaluate_rosenbrock)
generator = RandomGenerator(vocs=vocs)
X = Xopt(generator=generator, evaluator=evaluator)
X.random_evaluate(15)

# set values if testing import os from xopt import Xopt, Evaluator from xopt.generators import RandomGenerator from xopt.resources.test_functions.rosenbrock import ( evaluate_rosenbrock, make_rosenbrock_vocs, ) from xopt.generators.bayesian.visualize import visualize_model from xopt.generators.bayesian.models.standard import StandardModelConstructor # Ignore all warnings import warnings warnings.filterwarnings("ignore") SMOKE_TEST = os.environ.get("SMOKE_TEST") NUM_MC_SAMPLES = 1 if SMOKE_TEST else 128 NUM_RESTARTS = 1 if SMOKE_TEST else 20 # make rosenbrock function vocs in 3D vocs = make_rosenbrock_vocs(3) # collect some data using random sampling evaluator = Evaluator(function=evaluate_rosenbrock) generator = RandomGenerator(vocs=vocs) X = Xopt(generator=generator, evaluator=evaluator) X.random_evaluate(15)

Out[1]:

	x0	x1	x2	y	xopt_runtime	xopt_error
0	0.150312	-0.184042	1.329408	174.235180	0.000018	False
1	-1.726492	0.127469	-1.479221	1045.972236	0.000007	False
2	-1.629943	1.609928	-0.495462	1070.025832	0.000004	False
3	-1.068639	0.504390	0.484146	50.455986	0.000004	False
4	0.885640	-0.941160	-1.453589	848.788156	0.000004	False
5	0.392565	1.806848	-1.060040	2144.511849	0.000004	False
6	1.894799	-0.845450	0.980046	1978.797355	0.000004	False
7	-1.589873	-0.022197	-1.625279	922.261828	0.000003	False
8	0.533737	-0.070768	0.193499	17.565045	0.000004	False
9	1.512240	-0.141285	-0.526709	621.042874	0.000003	False
10	0.692413	-1.937287	1.695596	1016.101843	0.000003	False
11	0.851450	-0.413323	-1.502913	411.733873	0.000004	False
12	-0.941866	1.746577	-1.538299	2183.931458	0.000003	False
13	-0.424462	-0.817143	1.516642	176.860479	0.000003	False
14	1.109525	-0.302809	-1.279052	424.874455	0.000004	False

Standard Model Constructor¶

In [2]:

print(StandardModelConstructor.__doc__)

print(StandardModelConstructor.__doc__)

    A class for constructing independent models for each objective and constraint.

    Attributes
    ----------
    name : str
        The name of the model (frozen).

    use_low_noise_prior : bool
        Specify if the model should assume a low noise environment.

    covar_modules : Dict[str, Kernel]
        Covariance modules for GP models.

    mean_modules : Dict[str, Module]
        Prior mean modules for GP models.

    trainable_mean_keys : List[str]
        List of prior mean modules that can be trained.

    transform_inputs : Union[Dict[str, bool], bool]
        Specify if inputs should be transformed inside the GP model. Can optionally
        specify a dict of specifications.

    custom_noise_prior : Optional[Prior]
        Specify a custom noise prior for the GP likelihood. Overwrites value specified
        by use_low_noise_prior.

    use_cached_hyperparameters : Optional[bool]
        Flag to specify if cached hyperparameters should be used in model creation.
        Training will still occur unless train_model is False.

    train_method : Literal["lbfgs", "adam"]
        Numerical optimization algorithm to use.

    train_model : bool
        Flag to specify if the model should be trained (fitted to data).

    train_config : NumericalOptimizerConfig
        Configuration of the numerical optimizer.

    

Create GP model based on the data¶

In [3]:

data = X.data

data = X.data

In [4]:

model_constructor = StandardModelConstructor()

# here we build a model from vocs
model = model_constructor.build_model_from_vocs(vocs=vocs, data=data)

# here we build a model from info (more flexible)
model = model_constructor.build_model(
    input_names=["x0", "x1", "x2"], outcome_names=["y"], data=data
)
help(model_constructor.build_model)

model_constructor = StandardModelConstructor() # here we build a model from vocs model = model_constructor.build_model_from_vocs(vocs=vocs, data=data) # here we build a model from info (more flexible) model = model_constructor.build_model( input_names=["x0", "x1", "x2"], outcome_names=["y"], data=data ) help(model_constructor.build_model)

Help on method build_model in module xopt.generators.bayesian.models.standard:

build_model(input_names: List[str], outcome_names: List[str], data: pandas.DataFrame, input_bounds: Dict[str, List] = None, dtype: torch.dtype = torch.float64, device: Union[torch.device, str] = 'cpu') -> botorch.models.model_list_gp_regression.ModelListGP method of xopt.generators.bayesian.models.standard.StandardModelConstructor instance
    Construct independent models for each objective and constraint.

    Parameters
    ----------
    input_names : List[str]
        Names of input variables.
    outcome_names : List[str]
        Names of outcome variables.
    data : pd.DataFrame
        Data used for training the model.
    input_bounds : Dict[str, List], optional
        Bounds for input variables.
    dtype : torch.dtype, optional
        Data type for the model (default is torch.double).
    device : Union[torch.device, str], optional
        Device on which to perform computations (default is "cpu").

    Returns
    -------
    ModelListGP
        A list of trained botorch models.

Examine GP model hyperparameters¶

Here we look at the GP hyperparameters for the objective function (the first model). Note: the hyperparameters here are in raw_units (due to contraints on parameter values, ie. lengthscales > 0)

In [5]:

objective_model = model.models[vocs.output_names.index("y")]

# print raw hyperparameter values
for name, val in objective_model.named_parameters():
    print(name, val)

objective_model = model.models[vocs.output_names.index("y")] # print raw hyperparameter values for name, val in objective_model.named_parameters(): print(name, val)

likelihood.noise_covar.raw_noise Parameter containing:
tensor([0.0006], requires_grad=True)
mean_module.raw_constant Parameter containing:
tensor(0.4579, requires_grad=True)
covar_module.raw_lengthscale Parameter containing:
tensor([[0.4620, 0.2797, 0.5721]], requires_grad=True)

In [6]:

# print real values - note that these are in normalized coordinates
print("prior mean:", objective_model.mean_module.constant.data)
print("noise:", objective_model.likelihood.noise_covar.noise.data)
print("lengthscales", objective_model.covar_module.lengthscale.data)

# print real values - note that these are in normalized coordinates print("prior mean:", objective_model.mean_module.constant.data) print("noise:", objective_model.likelihood.noise_covar.noise.data) print("lengthscales", objective_model.covar_module.lengthscale.data)

prior mean: tensor(0.4579)
noise: tensor([0.0006])
lengthscales tensor([[0.4620, 0.2797, 0.5721]])

In [7]:

objective_model

objective_model

Out[7]:

SingleTaskGP(
  (likelihood): GaussianLikelihood(
    (noise_covar): HomoskedasticNoise(
      (noise_prior): GammaPrior()
      (raw_noise_constraint): GreaterThan(1.000E-04)
    )
  )
  (mean_module): ConstantMean()
  (covar_module): RBFKernel(
    (lengthscale_prior): LogNormalPrior()
    (raw_lengthscale_constraint): GreaterThan(2.500E-02)
  )
  (outcome_transform): Standardize()
  (input_transform): Normalize()
)

Visualize model predictions¶

In [8]:

fig, ax = visualize_model(
    model, vocs, data, variable_names=["x0", "x1"], reference_point={"x2": 0.0}
)

fig, ax = visualize_model( model, vocs, data, variable_names=["x0", "x1"], reference_point={"x2": 0.0} )

Using Custom Kernels via covar_modules¶

The covar_modules parameter in StandardModelConstructor allows you to specify custom GPyTorch kernels for specific outputs. This is useful when you have domain knowledge about the function structure.

In [9]:

from gpytorch.kernels import MaternKernel, ScaleKernel

# Example 1: Use a specific kernel for the objective
# Here we use a Matern kernel with nu=2.5 for the objective "y"
custom_covar_modules = {
    "y": ScaleKernel(MaternKernel(nu=2.5))  # Matern 5/2 kernel for objective
}

model_constructor_custom = StandardModelConstructor(
    covar_modules=custom_covar_modules, use_low_noise_prior=True
)

# Build model with custom kernel
custom_model = model_constructor_custom.build_model_from_vocs(vocs=vocs, data=data)

print("Custom model with Matern kernel:")
print(f"Kernel type: {custom_model.models[0].covar_module}")
print(f"Nu parameter: {custom_model.models[0].covar_module.base_kernel.nu}")

from gpytorch.kernels import MaternKernel, ScaleKernel # Example 1: Use a specific kernel for the objective # Here we use a Matern kernel with nu=2.5 for the objective "y" custom_covar_modules = { "y": ScaleKernel(MaternKernel(nu=2.5)) # Matern 5/2 kernel for objective } model_constructor_custom = StandardModelConstructor( covar_modules=custom_covar_modules, use_low_noise_prior=True ) # Build model with custom kernel custom_model = model_constructor_custom.build_model_from_vocs(vocs=vocs, data=data) print("Custom model with Matern kernel:") print(f"Kernel type: {custom_model.models[0].covar_module}") print(f"Nu parameter: {custom_model.models[0].covar_module.base_kernel.nu}")

Custom model with Matern kernel:
Kernel type: ScaleKernel(
  (base_kernel): MaternKernel(
    (raw_lengthscale_constraint): Positive()
  )
  (raw_outputscale_constraint): Positive()
)
Nu parameter: 2.5