Xopt Evaluator Basic Usage¶

The Evaluator handles the execution of the user-provided function with optional function_kwags, asyncrhonously and parallel, with exception handling.

In [1]:

from xopt import Xopt, Evaluator, VOCS
from xopt.generators.random import RandomGenerator

# Usage with a parallel executor.
from xopt import AsynchronousXopt

import pandas as pd

from time import sleep
from numpy.random import randint

from typing import Dict

import numpy as np

from concurrent.futures import ProcessPoolExecutor

# needed for macos
import platform

if platform.system() == "Darwin":
    import multiprocessing

    multiprocessing.set_start_method("fork")

from xopt import Xopt, Evaluator, VOCS from xopt.generators.random import RandomGenerator # Usage with a parallel executor. from xopt import AsynchronousXopt import pandas as pd from time import sleep from numpy.random import randint from typing import Dict import numpy as np from concurrent.futures import ProcessPoolExecutor # needed for macos import platform if platform.system() == "Darwin": import multiprocessing multiprocessing.set_start_method("fork")

In [2]:

np.random.seed(666)  # for reproducibility

np.random.seed(666) # for reproducibility

Define a custom function f(inputs: Dict) -> outputs: Dict.

In [3]:

def f(inputs: Dict, enable_errors=True) -> Dict:
    sleep(randint(1, 5) * 0.1)  # simulate computation time
    # Make some occasional errors
    if enable_errors and np.any(inputs["x"] > 0.8):
        raise ValueError("x > 0.8")

    return {"f1": inputs["x"] ** 2 + inputs["y"] ** 2}

def f(inputs: Dict, enable_errors=True) -> Dict: sleep(randint(1, 5) * 0.1) # simulate computation time # Make some occasional errors if enable_errors and np.any(inputs["x"] > 0.8): raise ValueError("x > 0.8") return {"f1": inputs["x"] ** 2 + inputs["y"] ** 2}

Define variables, objectives, constraints, and other settings (VOCS)

In [4]:

vocs = VOCS(variables={"x": [0, 1], "y": [0, 1]}, objectives={"f1": "MINIMIZE"})
vocs

vocs = VOCS(variables={"x": [0, 1], "y": [0, 1]}, objectives={"f1": "MINIMIZE"}) vocs

Out[4]:

VOCS(variables={'x': [0.0, 1.0], 'y': [0.0, 1.0]}, constraints={}, objectives={'f1': 'MINIMIZE'}, constants={}, observables=[])

This can be used to make some random inputs for evaluating the function.

In [5]:

in1 = vocs.random_inputs()[0]

f(in1, enable_errors=False)

in1 = vocs.random_inputs()[0] f(in1, enable_errors=False)

Out[5]:

{'f1': 0.11401572022703582}

In [6]:

# Add in occasional errors.
try:
    f({"x": 1, "y": 0})
except Exception as ex:
    print(f"Caught error in f: {ex}")

# Add in occasional errors. try: f({"x": 1, "y": 0}) except Exception as ex: print(f"Caught error in f: {ex}")

Caught error in f: x > 0.8

In [7]:

# Create Evaluator
ev = Evaluator(function=f)

# Create Evaluator ev = Evaluator(function=f)

In [8]:

# Single input evaluation
ev.evaluate(in1)

# Single input evaluation ev.evaluate(in1)

Out[8]:

{'f1': 0.11401572022703582,
 'xopt_runtime': 0.2001936350000051,
 'xopt_error': False}

In [9]:

# Dataframe evaluation
in10 = pd.DataFrame({"x": np.linspace(0, 1, 10), "y": np.linspace(0, 1, 10)})
ev.evaluate_data(in10)

# Dataframe evaluation in10 = pd.DataFrame({"x": np.linspace(0, 1, 10), "y": np.linspace(0, 1, 10)}) ev.evaluate_data(in10)

Out[9]:

	x	y	f1	xopt_runtime	xopt_error	xopt_error_str
0	0.000000	0.000000	0.000000	0.300169	False	NaN
1	0.111111	0.111111	0.024691	0.100183	False	NaN
2	0.222222	0.222222	0.098765	0.200190	False	NaN
3	0.333333	0.333333	0.222222	0.400207	False	NaN
4	0.444444	0.444444	0.395062	0.300183	False	NaN
5	0.555556	0.555556	0.617284	0.400215	False	NaN
6	0.666667	0.666667	0.888889	0.100157	False	NaN
7	0.777778	0.777778	1.209877	0.400167	False	NaN
8	0.888889	0.888889	NaN	0.100850	True	Traceback (most recent call last):\n File "/h...
9	1.000000	1.000000	NaN	0.300488	True	Traceback (most recent call last):\n File "/h...

In [10]:

# Dataframe evaluation, vectorized
ev.vectorized = True
ev.evaluate_data(in10)

# Dataframe evaluation, vectorized ev.vectorized = True ev.evaluate_data(in10)

Out[10]:

	x	y	xopt_runtime	xopt_error	xopt_error_str
0	0.000000	0.000000	0.200968	True	Traceback (most recent call last):\n File "/h...
1	0.111111	0.111111	0.200968	True	Traceback (most recent call last):\n File "/h...
2	0.222222	0.222222	0.200968	True	Traceback (most recent call last):\n File "/h...
3	0.333333	0.333333	0.200968	True	Traceback (most recent call last):\n File "/h...
4	0.444444	0.444444	0.200968	True	Traceback (most recent call last):\n File "/h...
5	0.555556	0.555556	0.200968	True	Traceback (most recent call last):\n File "/h...
6	0.666667	0.666667	0.200968	True	Traceback (most recent call last):\n File "/h...
7	0.777778	0.777778	0.200968	True	Traceback (most recent call last):\n File "/h...
8	0.888889	0.888889	0.200968	True	Traceback (most recent call last):\n File "/h...
9	1.000000	1.000000	0.200968	True	Traceback (most recent call last):\n File "/h...

Executors¶

In [11]:

MAX_WORKERS = 10

MAX_WORKERS = 10

In [12]:

# Create Executor instance
executor = ProcessPoolExecutor(max_workers=MAX_WORKERS)
executor

# Create Executor instance executor = ProcessPoolExecutor(max_workers=MAX_WORKERS) executor

Out[12]:

<concurrent.futures.process.ProcessPoolExecutor at 0x7efd35c1cd70>

In [13]:

# Dask (Optional)
# from dask.distributed import Client
# import logging
# client = Client( silence_logs=logging.ERROR)
# executor = client.get_executor()
# client

# Dask (Optional) # from dask.distributed import Client # import logging # client = Client( silence_logs=logging.ERROR) # executor = client.get_executor() # client

In [14]:

# This calls `executor.map`
ev = Evaluator(function=f, executor=executor, max_workers=MAX_WORKERS)

# This calls `executor.map` ev = Evaluator(function=f, executor=executor, max_workers=MAX_WORKERS)

In [15]:

# This will run in parallel
ev.evaluate_data(in10)

# This will run in parallel ev.evaluate_data(in10)

Out[15]:

	x	y	f1	xopt_runtime	xopt_error	xopt_error_str
0	0.000000	0.000000	0.000000	0.400483	False	NaN
1	0.111111	0.111111	0.024691	0.400421	False	NaN
2	0.222222	0.222222	0.098765	0.400415	False	NaN
3	0.333333	0.333333	0.222222	0.400389	False	NaN
4	0.444444	0.444444	0.395062	0.400839	False	NaN
5	0.555556	0.555556	0.617284	0.400436	False	NaN
6	0.666667	0.666667	0.888889	0.400600	False	NaN
7	0.777778	0.777778	1.209877	0.400534	False	NaN
8	0.888889	0.888889	NaN	0.401630	True	Traceback (most recent call last):\n File "/h...
9	1.000000	1.000000	NaN	0.401548	True	Traceback (most recent call last):\n File "/h...

Evaluator in the Xopt object¶

In [16]:

X = Xopt(
    generator=RandomGenerator(vocs=vocs), evaluator=Evaluator(function=f), vocs=vocs
)
X.strict = False

# Evaluate to the evaluator some new inputs
X.evaluate_data(X.vocs.random_inputs(4))

X = Xopt( generator=RandomGenerator(vocs=vocs), evaluator=Evaluator(function=f), vocs=vocs ) X.strict = False # Evaluate to the evaluator some new inputs X.evaluate_data(X.vocs.random_inputs(4))

Out[16]:

	x	y	f1	xopt_runtime	xopt_error	xopt_error_str
0	0.491934	0.299155	0.331493	0.100151	False	NaN
1	0.799752	0.706772	1.139131	0.100146	False	NaN
2	0.255846	0.225521	0.116317	0.100121	False	NaN
3	0.807108	0.994891	NaN	0.100786	True	Traceback (most recent call last):\n File "/h...

Asynchronous Xopt¶

Instead of waiting for evaluations to be finished, AsynchronousXopt can be used to generate candidates while waiting for other evaluations to finish (requires parallel execution). In this case, calling X.step() generates and executes a number of candidates that are executed in parallel using python concurrent.futures formalism. Calling X.step() again will generate and evaluate new points based on finished futures asynchronously.

In [17]:

executor = ProcessPoolExecutor(max_workers=MAX_WORKERS)

X2 = AsynchronousXopt(
    generator=RandomGenerator(vocs=vocs),
    evaluator=Evaluator(function=f, executor=executor, max_workers=MAX_WORKERS),
    vocs=vocs,
)
X2.strict = False

executor = ProcessPoolExecutor(max_workers=MAX_WORKERS) X2 = AsynchronousXopt( generator=RandomGenerator(vocs=vocs), evaluator=Evaluator(function=f, executor=executor, max_workers=MAX_WORKERS), vocs=vocs, ) X2.strict = False

In [18]:

X2.step()

X2.step()

In [19]:

for _ in range(20):
    X2.step()

len(X2.data)

for _ in range(20): X2.step() len(X2.data)

Out[19]:

41

In [20]:

X2.data.plot.scatter("x", "y")

X2.data.plot.scatter("x", "y")

Out[20]:

<Axes: xlabel='x', ylabel='y'>

In [21]:

# Asynchronous, Vectorized
X2 = AsynchronousXopt(
    generator=RandomGenerator(vocs=vocs),
    evaluator=Evaluator(function=f, executor=executor, max_workers=MAX_WORKERS),
    vocs=vocs,
)
X2.evaluator.vectorized = True
X2.strict = False

# This takes fewer steps to achieve a similar number of evaluations
for _ in range(3):
    X2.step()

len(X2.data)

# Asynchronous, Vectorized X2 = AsynchronousXopt( generator=RandomGenerator(vocs=vocs), evaluator=Evaluator(function=f, executor=executor, max_workers=MAX_WORKERS), vocs=vocs, ) X2.evaluator.vectorized = True X2.strict = False # This takes fewer steps to achieve a similar number of evaluations for _ in range(3): X2.step() len(X2.data)

Out[21]:

30