Xopt Parallel Examples¶

Xopt provides methods to parallelize optimizations using Processes, Threads, MPI, and Dask using the concurrent.futures interface as defined in https://www.python.org/dev/peps/pep-3148/ .

In [1]:

Copied!





# Helpers for this notebook
import multiprocessing
from concurrent.futures import ProcessPoolExecutor
from dask.distributed import Client

import matplotlib.pyplot as plt

import pandas as pd
from concurrent.futures import ThreadPoolExecutor

import os

from xopt import AsynchronousXopt as Xopt
from xopt.vocs import get_feasibility_data

SMOKE_TEST = os.environ.get("SMOKE_TEST")

# Notebook printing output
# from xopt import output_notebook
# output_notebook()

N_CPUS = multiprocessing.cpu_count()
N_CPUS

# directory for data.
os.makedirs("temp", exist_ok=True)
# Helpers for this notebook
import multiprocessing
from concurrent.futures import ProcessPoolExecutor
from dask.distributed import Client

import matplotlib.pyplot as plt

import pandas as pd
from concurrent.futures import ThreadPoolExecutor

import os

from xopt import AsynchronousXopt as Xopt
from xopt.vocs import get_feasibility_data

SMOKE_TEST = os.environ.get("SMOKE_TEST")

# Notebook printing output
# from xopt import output_notebook
# output_notebook()

N_CPUS = multiprocessing.cpu_count()
N_CPUS

# directory for data.
os.makedirs("temp", exist_ok=True)

The Xopt object can be instantiated from a JSON or YAML file, or a dict, with the proper structure.

Here we will make one

In [2]:

Copied!





# Make a proper input file.
MAX_EVALUATIONS = 32 if SMOKE_TEST else 1000
YAML = """

stopping_condition:
    name: MaxEvaluationsCondition
    max_evaluations: 1000

generator:
  name: cnsga
  output_path: temp
  population_size:  64
  vocs:
    variables:
      x1: [0, 3.14159]
      x2: [0, 3.14159]
    objectives: {y1: MINIMIZE, y2: MINIMIZE}
    constraints:
      c1: [GREATER_THAN, 0]
      c2: [LESS_THAN, 0.5]
    constants: {a: dummy_constant}
    
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    sleep: 0
    random_sleep: 0.1
  


"""
X = Xopt(YAML)
X.stopping_condition.max_evaluations = MAX_EVALUATIONS
X
# Make a proper input file.
MAX_EVALUATIONS = 32 if SMOKE_TEST else 1000
YAML = """

stopping_condition:
    name: MaxEvaluationsCondition
    max_evaluations: 1000

generator:
  name: cnsga
  output_path: temp
  population_size:  64
  vocs:
    variables:
      x1: [0, 3.14159]
      x2: [0, 3.14159]
    objectives: {y1: MINIMIZE, y2: MINIMIZE}
    constraints:
      c1: [GREATER_THAN, 0]
      c2: [LESS_THAN, 0.5]
    constants: {a: dummy_constant}
    
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    sleep: 0
    random_sleep: 0.1
  


"""
X = Xopt(YAML)
X.stopping_condition.max_evaluations = MAX_EVALUATIONS
X

Out[2]:

            Xopt
________________________________
Version: 0.1.dev1+ga7f787a86
Data size: 0
Config as YAML:
data_dump_file: null
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    raise_probability: 0
    random_sleep: 0.1
    sleep: 0
  max_workers: 1
  vectorized: false
generator:
  crossover_probability: 0.9
  mutation_probability: 1.0
  name: cnsga
  output_path: temp
  population: null
  population_file: null
  population_size: 64
  returns_id: false
  supports_constraints: true
  supports_multi_objective: true
  supports_single_objective: true
  vocs:
    constants:
      a:
        dtype: null
        type: Constant
        value: dummy_constant
    constraints:
      c1:
        dtype: null
        type: GreaterThanConstraint
        value: 0.0
      c2:
        dtype: null
        type: LessThanConstraint
        value: 0.5
    objectives:
      y1:
        dtype: null
        type: MinimizeObjective
      y2:
        dtype: null
        type: MinimizeObjective
    observables: {}
    variables:
      x1:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
      x2:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
is_done: false
serialize_inline: false
serialize_torch: false
stopping_condition:
  count_valid_only: false
  max_evaluations: 1000
  name: MaxEvaluationsCondition
  use_dataframe_index: false
strict: true
xopt_dump_file: null

In [3]:

Copied!

%%timeit
# Check that the average time is close to random_sleep
X.evaluator.function({"x1": 0.5, "x2": 0.5}, random_sleep=0.1)
%%timeit
# Check that the average time is close to random_sleep
X.evaluator.function({"x1": 0.5, "x2": 0.5}, random_sleep=0.1)

101 ms ± 17.8 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [4]:

Copied!

%%time
X.run()
%%time
X.run()

CPU times: user 5.83 s, sys: 36.1 ms, total: 5.87 s
Wall time: 1min 58s

Processes¶

In [5]:

Copied!





%%time
X = Xopt(YAML)

with ProcessPoolExecutor(max_workers=N_CPUS) as executor:
    X.evaluator.executor = executor
    X.evaluator.max_workers = N_CPUS
    X.run()
len(X.data)
%%time
X = Xopt(YAML)

with ProcessPoolExecutor(max_workers=N_CPUS) as executor:
    X.evaluator.executor = executor
    X.evaluator.max_workers = N_CPUS
    X.run()
len(X.data)

CPU times: user 5.79 s, sys: 185 ms, total: 5.97 s
Wall time: 28.6 s

Out[5]:

Threads¶

Continue running, this time with threads.

In [6]:

Copied!





%%time
X = Xopt(YAML)

with ThreadPoolExecutor(max_workers=N_CPUS) as executor:
    X.evaluator.executor = executor
    X.evaluator.max_workers = N_CPUS
    X.run()
len(X.data)
%%time
X = Xopt(YAML)

with ThreadPoolExecutor(max_workers=N_CPUS) as executor:
    X.evaluator.executor = executor
    X.evaluator.max_workers = N_CPUS
    X.run()
len(X.data)

CPU times: user 5.37 s, sys: 65.5 ms, total: 5.44 s
Wall time: 28.6 s

Out[6]:

MPI¶

The test.yaml file completely defines the problem. We will also direct the logging to an xopt.log file. The following invocation recruits 4 MPI workers to solve this problem.

We can also continue by calling .save with a JSON filename. This will write all of previous results into the file.

In [7]:

Copied!

X = Xopt(YAML)
X.dump("test.yaml")  # Write this input to file
!cat test.yaml
X = Xopt(YAML)
X.dump("test.yaml")  # Write this input to file
!cat test.yaml

data: null
data_dump_file: null
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    raise_probability: 0
    random_sleep: 0.1
    sleep: 0
  max_workers: 1
  vectorized: false
generator:
  crossover_probability: 0.9
  mutation_probability: 1.0
  name: cnsga
  output_path: temp
  population: null
  population_file: null
  population_size: 64
  returns_id: false
  supports_constraints: true
  supports_multi_objective: true
  supports_single_objective: true
  vocs:
    constants:
      a:
        dtype: null
        type: Constant
        value: dummy_constant
    constraints:
      c1:
        dtype: null
        type: GreaterThanConstraint
        value: 0.0
      c2:
        dtype: null
        type: LessThanConstraint
        value: 0.5
    objectives:
      y1:
        dtype: null
        type: MinimizeObjective
      y2:
        dtype: null
        type: MinimizeObjective
    observables: {}
    variables:
      x1:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
      x2:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
is_done: false
serialize_inline: false
serialize_torch: false
stopping_condition:
  count_valid_only: false
  max_evaluations: 1000
  name: MaxEvaluationsCondition
  use_dataframe_index: false
strict: true
xopt_dump_file: null

In [8]:

Copied!

%%time
!mpirun -n 8 xopt-mpirun -vv --logfile xopt.log test.yaml
%%time
!mpirun -n 8 xopt-mpirun -vv --logfile xopt.log test.yaml

Namespace(input_file='test.yaml', logfile='xopt.log', verbose=2, asynchronous=True, python_path=[], override=[], initial_data=None)

Python path additions: ['/home/runner/work/Xopt/Xopt/docs/examples/basic']
Parallel execution with 8 workers
Python path additions: ['/home/runner/work/Xopt/Xopt/docs/examples/basic']
Enabling async mode
Parallel execution with 8 workers
Enabling async mode
Initialized generator cnsga
Created toolbox with 2 variables, 2 constraints, and 2 objectives.
    Using selection algorithm: nsga2

Initialized generator cnsga
Created toolbox with 2 variables, 2 constraints, and 2 objectives.
    Using selection algorithm: nsga2

Python path additions: ['/home/runner/work/Xopt/Xopt/docs/examples/basic']
Parallel execution with 8 workers
Enabling async mode
Python path additions: ['/home/runner/work/Xopt/Xopt/docs/examples/basic']
Parallel execution with 8 workers
Enabling async mode
Python path additions: ['/home/runner/work/Xopt/Xopt/docs/examples/basic']
Parallel execution with 8 workers
Enabling async mode

Python path additions: ['/home/runner/work/Xopt/Xopt/docs/examples/basic']
Parallel execution with 8 workers
Enabling async mode
Initialized generator cnsga
Created toolbox with 2 variables, 2 constraints, and 2 objectives.
    Using selection algorithm: nsga2

Initialized generator cnsga
Created toolbox with 2 variables, 2 constraints, and 2 objectives.
    Using selection algorithm: nsga2
Python path additions: ['/home/runner/work/Xopt/Xopt/docs/examples/basic']
Parallel execution with 8 workers
Enabling async mode
Python path additions: ['/home/runner/work/Xopt/Xopt/docs/examples/basic']
Parallel execution with 8 workers
Enabling async mode
Initialized generator cnsga
Created toolbox with 2 variables, 2 constraints, and 2 objectives.
    Using selection algorithm: nsga2

            Xopt
________________________________
Version: 0.1.dev1+ga7f787a86
Data size: 0
Config as YAML:
data_dump_file: null
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    raise_probability: 0
    random_sleep: 0.1
    sleep: 0
  max_workers: 1
  vectorized: false
generator:
  crossover_probability: 0.9
  mutation_probability: 1.0
  name: cnsga
  output_path: temp
  population: null
  population_file: null
  population_size: 64
  returns_id: false
  supports_constraints: true
  supports_multi_objective: true
  supports_single_objective: true
  vocs:
    constants:
      a:
        dtype: null
        type: Constant
        value: dummy_constant
    constraints:
      c1:
        dtype: null
        type: GreaterThanConstraint
        value: 0.0
      c2:
        dtype: null
        type: LessThanConstraint
        value: 0.5
    objectives:
      y1:
        dtype: null
        type: MinimizeObjective
      y2:
        dtype: null
        type: MinimizeObjective
    observables: {}
    variables:
      x1:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
      x2:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
is_done: false
serialize_inline: false
serialize_torch: false
stopping_condition:
  count_valid_only: false
  max_evaluations: 1000
  name: MaxEvaluationsCondition
  use_dataframe_index: false
strict: true
xopt_dump_file: null


Initialized generator cnsga
Created toolbox with 2 variables, 2 constraints, and 2 objectives.
    Using selection algorithm: nsga2
Running Xopt

Initialized generator cnsga
Created toolbox with 2 variables, 2 constraints, and 2 objectives.
Initialized generator cnsga
Created toolbox with 2 variables, 2 constraints, and 2 objectives.
    Using selection algorithm: nsga2
    Using selection algorithm: nsga2

Xopt is done. Stopping condition met.

CPU times: user 326 ms, sys: 63.6 ms, total: 390 ms
Wall time: 31.2 s

In [9]:

Copied!

!tail xopt.log
!tail xopt.log

2026-10-30T19:34:38+0000 - xopt.generators.ga.cnsga - INFO - Created toolbox with 2 variables, 2 constraints, and 2 objectives.
2026-10-30T19:34:38+0000 - xopt.generators.ga.cnsga - INFO -     Using selection algorithm: nsga2
2026-10-30T19:34:38+0000 - xopt.base - INFO - Running Xopt
2026-10-30T19:34:38+0000 - xopt.generator - INFO - Initialized generator cnsga
2026-10-30T19:34:38+0000 - xopt.generators.ga.cnsga - INFO - Created toolbox with 2 variables, 2 constraints, and 2 objectives.
2026-10-30T19:34:38+0000 - xopt.generator - INFO - Initialized generator cnsga
2026-10-30T19:34:38+0000 - xopt.generators.ga.cnsga - INFO - Created toolbox with 2 variables, 2 constraints, and 2 objectives.
2026-10-30T19:34:38+0000 - xopt.generators.ga.cnsga - INFO -     Using selection algorithm: nsga2
2026-10-30T19:34:38+0000 - xopt.generators.ga.cnsga - INFO -     Using selection algorithm: nsga2
2026-10-30T19:34:55+0000 - xopt.base - INFO - Xopt is done. Stopping condition met.

Dask¶

In [10]:

Copied!

client = Client()
executor = client.get_executor()
client
client = Client()
executor = client.get_executor()
client

Out[10]:

Client

Client-fdf3d806-d498-11f1-8d2b-3d83cfde6149

Connection method: Cluster object	Cluster type: distributed.LocalCluster
Dashboard: http://127.0.0.1:8787/status

Cluster Info

LocalCluster

626ebf56

Dashboard: http://127.0.0.1:8787/status	Workers: 4
Total threads: 4	Total memory: 15.61 GiB
Status: running	Using processes: True

Scheduler Info

Scheduler

Scheduler-c5a94515-d6a8-40d4-88d4-3d5cc06492df

Comm: tcp://127.0.0.1:34173	Workers: 0
Dashboard: http://127.0.0.1:8787/status	Total threads: 0
Started: Just now	Total memory: 0 B

Workers

Worker: 0

Comm: tcp://127.0.0.1:34297	Total threads: 1
Dashboard: http://127.0.0.1:34285/status	Memory: 3.90 GiB
Nanny: tcp://127.0.0.1:43787
Local directory: /tmp/dask-scratch-space/worker-15_jpr0w

Worker: 1

Comm: tcp://127.0.0.1:46537	Total threads: 1
Dashboard: http://127.0.0.1:34151/status	Memory: 3.90 GiB
Nanny: tcp://127.0.0.1:46691
Local directory: /tmp/dask-scratch-space/worker-6g38z7vs

Worker: 2

Comm: tcp://127.0.0.1:39127	Total threads: 1
Dashboard: http://127.0.0.1:40731/status	Memory: 3.90 GiB
Nanny: tcp://127.0.0.1:41993
Local directory: /tmp/dask-scratch-space/worker-apylh8xg

Worker: 3

Comm: tcp://127.0.0.1:34401	Total threads: 1
Dashboard: http://127.0.0.1:41079/status	Memory: 3.90 GiB
Nanny: tcp://127.0.0.1:40461
Local directory: /tmp/dask-scratch-space/worker-f165jgoa

In [11]:

Copied!





%%time
X = Xopt(YAML)
X.evaluator.executor = executor
X.evaluator.max_workers = N_CPUS
X.run()
len(X.data)
%%time
X = Xopt(YAML)
X.evaluator.executor = executor
X.evaluator.max_workers = N_CPUS
X.run()
len(X.data)

CPU times: user 11.3 s, sys: 959 ms, total: 12.3 s
Wall time: 36.8 s

Out[11]:

Load output into Pandas¶

This algorithm writes two types of files: gen_{i}.json with all of the new individuals evaluated in a generation, and pop_{i}.json with the latest best population. Xopt provides some functions to load these easily into a Pandas dataframe for further analysis.

In [12]:

Copied!

X.data
X.data

Out[12]:

	x1	x2	a	y1	y2	c1	c2	xopt_runtime	xopt_error
0	1.305791	1.168126	dummy_constant	1.305791	1.168126	2.006623	1.095691	0.085017	False
1	0.500686	0.444524	dummy_constant	0.500686	0.444524	-0.609916	0.003078	0.174499	False
2	0.790085	0.777294	dummy_constant	0.790085	0.777294	0.129272	0.161041	0.018458	False
3	1.077576	0.089774	dummy_constant	1.077576	0.089774	0.145373	0.501880	0.013584	False
4	1.127693	0.992430	dummy_constant	1.127693	0.992430	1.204222	0.636486	0.118189	False
...	...	...	...	...	...	...	...	...	...
995	0.866075	0.479840	dummy_constant	0.866075	0.479840	0.004204	0.134417	0.070830	False
996	1.067480	0.543825	dummy_constant	1.067480	0.543825	0.404263	0.323954	0.131139	False
997	0.758737	0.737104	dummy_constant	0.758737	0.737104	0.021668	0.123163	0.024134	False
998	0.866075	0.423809	dummy_constant	0.866075	0.423809	-0.124483	0.139816	0.172664	False
999	0.936178	0.141240	dummy_constant	0.936178	0.141240	-0.030165	0.318960	0.142068	False

1000 rows × 9 columns

In [13]:

Copied!

df = pd.concat([X.data, get_feasibility_data(X.vocs, X.data)], axis=1)
df[df["feasible"]]
df = pd.concat([X.data, get_feasibility_data(X.vocs, X.data)], axis=1)
df[df["feasible"]]

Out[13]:

	x1	x2	a	y1	y2	c1	c2	xopt_runtime	xopt_error	feasible_c1	feasible_c2	feasible
2	0.790085	0.777294	dummy_constant	0.790085	0.777294	0.129272	0.161041	0.018458	False	True	True	True
36	0.700541	0.913244	dummy_constant	0.700541	0.913244	0.374977	0.210987	0.082237	False	True	True	True
44	0.618111	1.195466	dummy_constant	0.618111	1.195466	0.789482	0.497623	0.152606	False	True	True	True
67	0.689219	0.782340	dummy_constant	0.689219	0.782340	0.033990	0.115520	0.065809	False	True	True	True
127	1.127864	0.311295	dummy_constant	1.127864	0.311295	0.408256	0.429823	0.075779	False	True	True	True
...	...	...	...	...	...	...	...	...	...	...	...	...
993	0.801557	0.543825	dummy_constant	0.801557	0.543825	0.037599	0.092857	0.054934	False	True	True	True
994	0.873806	0.836161	dummy_constant	0.873806	0.836161	0.368840	0.252735	0.088890	False	True	True	True
995	0.866075	0.479840	dummy_constant	0.866075	0.479840	0.004204	0.134417	0.070830	False	True	True	True
996	1.067480	0.543825	dummy_constant	1.067480	0.543825	0.404263	0.323954	0.131139	False	True	True	True
997	0.758737	0.737104	dummy_constant	0.758737	0.737104	0.021668	0.123163	0.024134	False	True	True	True

472 rows × 12 columns

In [14]:

Copied!

# Plot the feasible ones
feasible_df = df[df["feasible"]]
feasible_df.plot("y1", "y2", kind="scatter").set_aspect("equal")
# Plot the feasible ones
feasible_df = df[df["feasible"]]
feasible_df.plot("y1", "y2", kind="scatter").set_aspect("equal")

No description has been provided for this image

In [15]:

Copied!

# Plot the infeasible ones
infeasible_df = df[~df["feasible"]]
infeasible_df.plot("y1", "y2", kind="scatter").set_aspect("equal")
# Plot the infeasible ones
infeasible_df = df[~df["feasible"]]
infeasible_df.plot("y1", "y2", kind="scatter").set_aspect("equal")

In [16]:

Copied!

# This is the final population
df1 = X.generator.population
df1.plot("y1", "y2", kind="scatter").set_aspect("equal")
# This is the final population
df1 = X.generator.population
df1.plot("y1", "y2", kind="scatter").set_aspect("equal")

matplotlib plotting¶

You can always use matplotlib for customizable plotting

In [17]:

Copied!





# Extract objectives from output
k1, k2 = "y1", "y2"

fig, ax = plt.subplots(figsize=(6, 6))

ax.scatter(
    infeasible_df[k1],
    infeasible_df[k2],
    color="blue",
    marker=".",
    alpha=0.5,
    label="infeasible",
)
ax.scatter(
    feasible_df[k1], feasible_df[k2], color="orange", marker=".", label="feasible"
)
ax.scatter(df1[k1], df1[k2], color="red", marker=".", label="final population")
ax.set_xlabel(k1)
ax.set_ylabel(k2)
ax.set_aspect("auto")
ax.set_title("Xopt's CNSGA algorithm")
plt.legend()
# Extract objectives from output
k1, k2 = "y1", "y2"

fig, ax = plt.subplots(figsize=(6, 6))

ax.scatter(
    infeasible_df[k1],
    infeasible_df[k2],
    color="blue",
    marker=".",
    alpha=0.5,
    label="infeasible",
)
ax.scatter(
    feasible_df[k1], feasible_df[k2], color="orange", marker=".", label="feasible"
)
ax.scatter(df1[k1], df1[k2], color="red", marker=".", label="final population")
ax.set_xlabel(k1)
ax.set_ylabel(k2)
ax.set_aspect("auto")
ax.set_title("Xopt's CNSGA algorithm")
plt.legend()

Out[17]:

<matplotlib.legend.Legend at 0x75174848eea0>

In [18]:

Copied!





# Cleanup
#!rm -r dask-worker-space
!rm -r temp
!rm xopt.log*
!rm test.yaml
# Cleanup
#!rm -r dask-worker-space
!rm -r temp
!rm xopt.log*
!rm test.yaml