Nelder-Mead Generator adapted from SciPy¶

Most of the algorithms in scipy.optimize are self-contained functions that operate on the user-provided func. Xopt has adapted the Nelder-Mead directly from scipy.optimize to be in a generator form. This allows for the manual stepping through the algorithm.

In [1]:

from xopt.generators.sequential.neldermead import NelderMeadGenerator
from xopt import Evaluator, VOCS
from xopt.resources.test_functions.rosenbrock import rosenbrock

import pandas as pd

from xopt import Xopt
import numpy as np

from scipy.optimize import fmin

# from xopt import output_notebook
# output_notebook()

import matplotlib.pyplot as plt

from xopt.generators.sequential.neldermead import NelderMeadGenerator from xopt import Evaluator, VOCS from xopt.resources.test_functions.rosenbrock import rosenbrock import pandas as pd from xopt import Xopt import numpy as np from scipy.optimize import fmin # from xopt import output_notebook # output_notebook() import matplotlib.pyplot as plt

Nelder-Mead optimization of the Rosenbrock function with Xopt¶

In [2]:

YAML = """
max_evaluations: 500
generator:
  name: neldermead
  adaptive: true
evaluator:
  function: xopt.resources.test_functions.rosenbrock.evaluate_rosenbrock
vocs:
  variables:
    x0: [-5, 5]
    x1: [-5, 5]
  objectives: {y: MINIMIZE}
"""
X = Xopt.from_yaml(YAML)

YAML = """ max_evaluations: 500 generator: name: neldermead adaptive: true evaluator: function: xopt.resources.test_functions.rosenbrock.evaluate_rosenbrock vocs: variables: x0: [-5, 5] x1: [-5, 5] objectives: {y: MINIMIZE} """ X = Xopt.from_yaml(YAML)

In [3]:

XMIN = [1, 1]  # True minimum

XMIN = [1, 1] # True minimum

In [4]:

X.random_evaluate(2)
X.run()
X.data

X.random_evaluate(2) X.run() X.data

Out[4]:

	x0	x1	y	xopt_runtime	xopt_error
0	-1.903097	-2.311215	3528.470840	0.000008	False
1	0.437598	0.779478	34.889091	0.000005	False
2	0.437598	0.779478	34.889091	0.000006	False
3	0.459478	0.779478	32.595297	0.000006	False
4	0.437598	0.818452	39.624213	0.000005	False
...	...	...	...	...	...
495	1.000000	1.000000	0.000000	0.000005	False
496	1.000000	1.000000	0.000000	0.000005	False
497	1.000000	1.000000	0.000000	0.000005	False
498	1.000000	1.000000	0.000000	0.000005	False
499	1.000000	1.000000	0.000000	0.000005	False

500 rows × 5 columns

In [5]:

# Evaluation progression
X.data["y"].plot()
plt.yscale("log")
plt.xlabel("iteration")
plt.ylabel("Rosenbrock value")

# Evaluation progression X.data["y"].plot() plt.yscale("log") plt.xlabel("iteration") plt.ylabel("Rosenbrock value")

Out[5]:

Text(0, 0.5, 'Rosenbrock value')

In [6]:

# Minimum
dict(X.data.iloc[X.data["y"].argmin()])

# Minimum dict(X.data.iloc[X.data["y"].argmin()])

Out[6]:

{'x0': np.float64(1.0),
 'x1': np.float64(1.0),
 'y': np.float64(0.0),
 'xopt_runtime': np.float64(5.089000069347094e-06),
 'xopt_error': np.False_}

Visualize¶

In [7]:

fig, ax = plt.subplots(figsize=(8, 8))

Xgrid, Ygrid = np.meshgrid(np.linspace(-2, 2, 201), np.linspace(-2, 2, 201))

Zgrid = np.vectorize(lambda x, y: rosenbrock([x, y]))(Xgrid, Ygrid)
Zgrid = np.log(Zgrid + 1)

ax.pcolormesh(Xgrid, Ygrid, Zgrid)
ax.contour(Xgrid, Ygrid, Zgrid, levels=10, colors="black")
ax.set_xlabel("x0")
ax.set_ylabel("x1")


# Add all evaluations
ax.plot(X.data["x0"], X.data["x1"], color="red", alpha=0.5, marker=".")
ax.scatter(XMIN[0], XMIN[1], 50, marker="o", color="orange", label="True minimum")
ax.set_xlim(-2, 2)
ax.set_ylim(-2, 2)
# plt.legend()
ax.set_title("Xopt's Nelder-Mead progression")

fig, ax = plt.subplots(figsize=(8, 8)) Xgrid, Ygrid = np.meshgrid(np.linspace(-2, 2, 201), np.linspace(-2, 2, 201)) Zgrid = np.vectorize(lambda x, y: rosenbrock([x, y]))(Xgrid, Ygrid) Zgrid = np.log(Zgrid + 1) ax.pcolormesh(Xgrid, Ygrid, Zgrid) ax.contour(Xgrid, Ygrid, Zgrid, levels=10, colors="black") ax.set_xlabel("x0") ax.set_ylabel("x1") # Add all evaluations ax.plot(X.data["x0"], X.data["x1"], color="red", alpha=0.5, marker=".") ax.scatter(XMIN[0], XMIN[1], 50, marker="o", color="orange", label="True minimum") ax.set_xlim(-2, 2) ax.set_ylim(-2, 2) # plt.legend() ax.set_title("Xopt's Nelder-Mead progression")

Out[7]:

Text(0.5, 1.0, "Xopt's Nelder-Mead progression")

In [8]:

# Manually step the algorithm and collect simplexes
X = Xopt.from_yaml(YAML)
X.random_evaluate(1)
simplexes = []
for i in range(500):
    X.step()
    simplexes.append(X.generator.simplex)

# Manually step the algorithm and collect simplexes X = Xopt.from_yaml(YAML) X.random_evaluate(1) simplexes = [] for i in range(500): X.step() simplexes.append(X.generator.simplex)

In [9]:

def plot_simplex(simplex, ax=None):
    x0 = simplex["x0"]
    x1 = simplex["x1"]
    x0 = np.append(x0, x0[0])
    x1 = np.append(x1, x1[0])
    ax.plot(x0, x1)


fig, ax = plt.subplots(figsize=(8, 8))
ax.pcolormesh(Xgrid, Ygrid, Zgrid)
# ax.contour(Xgrid, Ygrid, Zgrid, levels=10, colors='black')
ax.set_xlim(-2, 2)
ax.set_ylim(-2, 2)
ax.set_xlabel("x0")
ax.set_ylabel("x1")
ax.set_title("Nelder-Mead simplex progression")

ax.scatter(XMIN[0], XMIN[1], 50, marker="o", color="orange", label="True minimum")

for simplex in simplexes:
    plot_simplex(simplex, ax)

def plot_simplex(simplex, ax=None): x0 = simplex["x0"] x1 = simplex["x1"] x0 = np.append(x0, x0[0]) x1 = np.append(x1, x1[0]) ax.plot(x0, x1) fig, ax = plt.subplots(figsize=(8, 8)) ax.pcolormesh(Xgrid, Ygrid, Zgrid) # ax.contour(Xgrid, Ygrid, Zgrid, levels=10, colors='black') ax.set_xlim(-2, 2) ax.set_ylim(-2, 2) ax.set_xlabel("x0") ax.set_ylabel("x1") ax.set_title("Nelder-Mead simplex progression") ax.scatter(XMIN[0], XMIN[1], 50, marker="o", color="orange", label="True minimum") for simplex in simplexes: plot_simplex(simplex, ax)

Compare with scipy.optimize.fmin Nelder-Mead¶

Notice that fmin is much faster here. This is because the function runs very fast, so the internal Xopt bookkeeping overhead dominates.

In [10]:

result = fmin(rosenbrock, [-1, -1])
result

result = fmin(rosenbrock, [-1, -1]) result

Optimization terminated successfully.
         Current function value: 0.000000
         Iterations: 67
         Function evaluations: 125

Out[10]:

array([0.99999886, 0.99999542])

In [11]:

X = Xopt.from_yaml(YAML)
X.random_evaluate(1)

X = Xopt.from_yaml(YAML) X.random_evaluate(1)

Out[11]:

	x0	x1	y	xopt_runtime	xopt_error
0	3.644546	3.885223	8838.275651	0.000006	False

In [12]:

X.run()
# Almost exactly the same number evaluations.
len(X.data)

X.run() # Almost exactly the same number evaluations. len(X.data)

Out[12]:

500

In [13]:

# results are the same
xbest = X.data.iloc[X.data["y"].argmin()]
xbest["x0"] == result[0], xbest["x1"] == result[1]

# results are the same xbest = X.data.iloc[X.data["y"].argmin()] xbest["x0"] == result[0], xbest["x1"] == result[1]

Out[13]:

(np.False_, np.False_)

NelderMeadGenerator object¶

In [14]:

NelderMeadGenerator.model_fields

NelderMeadGenerator.model_fields

Out[14]:

{'supports_batch_generation': FieldInfo(annotation=bool, required=False, default=False, description='flag that describes if this generator can generate batches of points', exclude=True, frozen=True),
 'supports_multi_objective': FieldInfo(annotation=bool, required=False, default=False, description='flag that describes if this generator can solve multi-objective problems', exclude=True, frozen=True),
 'supports_single_objective': FieldInfo(annotation=bool, required=False, default=True),
 'supports_constraints': FieldInfo(annotation=bool, required=False, default=False, description='flag that describes if this generator can solve constrained optimization problems', exclude=True, frozen=True),
 'vocs': FieldInfo(annotation=VOCS, required=True, description='generator VOCS', exclude=True),
 'data': FieldInfo(annotation=Union[DataFrame, NoneType], required=False, default=None, description='generator data', exclude=True),
 'is_active': FieldInfo(annotation=bool, required=False, default=False),
 'initial_point': FieldInfo(annotation=Union[Dict[str, float], NoneType], required=False, default=None),
 'initial_simplex': FieldInfo(annotation=Union[Dict[str, Union[List[float], ndarray]], NoneType], required=False, default=None),
 'adaptive': FieldInfo(annotation=bool, required=False, default=True, description='Change hyperparameters based on dimensionality'),
 'current_state': FieldInfo(annotation=SimplexState, required=False, default=SimplexState(astg=-1, N=None, kend=0, jend=0, ind=None, sim=None, fsim=None, fxr=None, x=None, xr=None, xe=None, xc=None, xcc=None, xbar=None, doshrink=0, ngen=0)),
 'future_state': FieldInfo(annotation=Union[SimplexState, NoneType], required=False, default=None),
 'x': FieldInfo(annotation=Union[ndarray, NoneType], required=False, default=None),
 'y': FieldInfo(annotation=Union[float, NoneType], required=False, default=None),
 'manual_data_cnt': FieldInfo(annotation=int, required=False, default=0, description='How many points are considered manual/not part of simplex run')}

In [15]:

Xbest = [33, 44]


def f(inputs, verbose=False):
    if verbose:
        print(f"evaluate f({inputs})")
    x0 = inputs["x0"]
    x1 = inputs["x1"]

    # if x0 < 10:
    #    raise ValueError('test XXXX')

    y = (x0 - Xbest[0]) ** 2 + (x1 - Xbest[1]) ** 2

    return {"y": y}


ev = Evaluator(function=f)
vocs = VOCS(
    variables={"x0": [-100, 100], "x1": [-100, 100]}, objectives={"y": "MINIMIZE"}
)
vocs.json()

Xbest = [33, 44] def f(inputs, verbose=False): if verbose: print(f"evaluate f({inputs})") x0 = inputs["x0"] x1 = inputs["x1"] # if x0 < 10: # raise ValueError('test XXXX') y = (x0 - Xbest[0]) ** 2 + (x1 - Xbest[1]) ** 2 return {"y": y} ev = Evaluator(function=f) vocs = VOCS( variables={"x0": [-100, 100], "x1": [-100, 100]}, objectives={"y": "MINIMIZE"} ) vocs.json()

Out[15]:

'{"variables":{"x0":[-100.0,100.0],"x1":[-100.0,100.0]},"constraints":{},"objectives":{"y":"MINIMIZE"},"constants":{},"observables":[]}'

In [16]:

# check output
f(vocs.random_inputs()[0])

# check output f(vocs.random_inputs()[0])

Out[16]:

{'y': 6772.123770236644}

In [17]:

G = NelderMeadGenerator(vocs=vocs, initial_point={"x0": 0, "x1": 0})
inputs = G.generate(1)
inputs

G = NelderMeadGenerator(vocs=vocs, initial_point={"x0": 0, "x1": 0}) inputs = G.generate(1) inputs

Out[17]:

[{'x0': np.float64(0.0), 'x1': np.float64(0.0)}]

In [18]:

# Further generate calls will continue to produce same point, as with BO
G.generate(1)

# Further generate calls will continue to produce same point, as with BO G.generate(1)

Out[18]:

[{'x0': np.float64(0.0), 'x1': np.float64(0.0)}]

In [19]:

ev.evaluate(inputs[0])

ev.evaluate(inputs[0])

Out[19]:

{'y': np.float64(3025.0),
 'xopt_runtime': 4.006999915873166e-06,
 'xopt_error': False}

In [20]:

# Adding new data will advance state to next step, and next generate() will yield new point
G.add_data(pd.DataFrame(inputs[0] | ev.evaluate(inputs[0]), index=[0]))
G.generate(1)

# Adding new data will advance state to next step, and next generate() will yield new point G.add_data(pd.DataFrame(inputs[0] | ev.evaluate(inputs[0]), index=[0])) G.generate(1)

Out[20]:

[{'x0': np.float64(0.00025), 'x1': np.float64(0.0)}]

In [21]:

# Create Xopt object
X = Xopt(
    evaluator=ev,
    vocs=vocs,
    generator=NelderMeadGenerator(vocs=vocs),
    max_evaluations=100,
)

# Optional: give an initial pioint
X.generator.initial_point = {"x0": 0, "x1": 0}

# Create Xopt object X = Xopt( evaluator=ev, vocs=vocs, generator=NelderMeadGenerator(vocs=vocs), max_evaluations=100, ) # Optional: give an initial pioint X.generator.initial_point = {"x0": 0, "x1": 0}

In [22]:

X.run()

X.run()

In [23]:

# This shows the latest simplex
X.generator.simplex

# This shows the latest simplex X.generator.simplex

Out[23]:

{'x0': array([19.32551212, 18.81564233, 13.99270888]),
 'x1': array([44.99152946, 41.9372879 , 45.9516701 ])}

In [24]:

X.data["y"].plot()
plt.yscale("log")

X.data["y"].plot() plt.yscale("log")

In [25]:

fig, ax = plt.subplots()
X.data.plot("x0", "x1", ax=ax, color="black", alpha=0.5)
ax.scatter(Xbest[0], Xbest[1], marker="x", color="red")

fig, ax = plt.subplots() X.data.plot("x0", "x1", ax=ax, color="black", alpha=0.5) ax.scatter(Xbest[0], Xbest[1], marker="x", color="red")

Out[25]:

<matplotlib.collections.PathCollection at 0x7f0c82cbf110>

In [26]:

# This is the raw internal state of the generator
a = X.generator.current_state
a

# This is the raw internal state of the generator a = X.generator.current_state a

Out[26]:

SimplexState(astg=2, N=2, kend=3, jend=0, ind=array([0., 2., 1.]), sim=array([[19.32551212, 44.99152946],
       [18.81564233, 41.9372879 ],
       [13.99270888, 45.9516701 ]]), fsim=array([187.97474953, 205.45078365, 365.08613186]), fxr=87.4876545585393, x=array(nan), xr=array([24.14844557, 40.97714726]), xe=array([29.22631391, 38.48988583]), xc=array([ 9.13505685, 49.06484781]), xcc=array([ 9.06650876, 48.69320702]), xbar=array([19.07057722, 43.46440868]), doshrink=0, ngen=100)

5-dimensional Rosenbrock¶

evaluate_rosenbrock works for arbitrary dimensions, so adding more variables to vocs transforms this problem.

In [27]:

YAML = """
max_evaluations: 500
generator:
  name: neldermead
evaluator:
  function: xopt.resources.test_functions.rosenbrock.evaluate_rosenbrock
vocs:
  variables:
    x1: [-5, 5]
    x2: [-5, 5]
    x3: [-5, 5]
    x4: [-5, 5]
    x5: [-5, 5]
  objectives:
    y: MINIMIZE
"""
X = Xopt.from_yaml(YAML)

YAML = """ max_evaluations: 500 generator: name: neldermead evaluator: function: xopt.resources.test_functions.rosenbrock.evaluate_rosenbrock vocs: variables: x1: [-5, 5] x2: [-5, 5] x3: [-5, 5] x4: [-5, 5] x5: [-5, 5] objectives: y: MINIMIZE """ X = Xopt.from_yaml(YAML)

In [28]:

X.random_evaluate(1)
X.run()
X.data["y"].plot()
plt.yscale("log")

X.random_evaluate(1) X.run() X.data["y"].plot() plt.yscale("log")

In [29]:

fig, ax = plt.subplots(figsize=(8, 8))

Xgrid, Ygrid = np.meshgrid(np.linspace(-2, 2, 201), np.linspace(-2, 2, 201))

Zgrid = np.vectorize(lambda x, y: rosenbrock([x, y, 1, 1, 1]))(
    Xgrid, Ygrid
)  # The minimum is at 1,1,1,1,1
Zgrid = np.log(Zgrid + 1)

ax.pcolormesh(Xgrid, Ygrid, Zgrid)
ax.contour(Xgrid, Ygrid, Zgrid, levels=10, colors="black")
ax.set_xlabel("x0")
ax.set_ylabel("x1")


# Add all evaluations
ax.plot(X.data["x1"], X.data["x2"], color="red", alpha=0.5, marker=".")
ax.scatter(XMIN[0], XMIN[1], 50, marker="o", color="orange", label="True minimum")
ax.set_xlim(-2, 2)
ax.set_ylim(-2, 2)
# plt.legend()
ax.set_title("Xopt's Nelder-Mead progression")

fig, ax = plt.subplots(figsize=(8, 8)) Xgrid, Ygrid = np.meshgrid(np.linspace(-2, 2, 201), np.linspace(-2, 2, 201)) Zgrid = np.vectorize(lambda x, y: rosenbrock([x, y, 1, 1, 1]))( Xgrid, Ygrid ) # The minimum is at 1,1,1,1,1 Zgrid = np.log(Zgrid + 1) ax.pcolormesh(Xgrid, Ygrid, Zgrid) ax.contour(Xgrid, Ygrid, Zgrid, levels=10, colors="black") ax.set_xlabel("x0") ax.set_ylabel("x1") # Add all evaluations ax.plot(X.data["x1"], X.data["x2"], color="red", alpha=0.5, marker=".") ax.scatter(XMIN[0], XMIN[1], 50, marker="o", color="orange", label="True minimum") ax.set_xlim(-2, 2) ax.set_ylim(-2, 2) # plt.legend() ax.set_title("Xopt's Nelder-Mead progression")

Out[29]:

Text(0.5, 1.0, "Xopt's Nelder-Mead progression")

In [ ]: