Xopt Parallel Examples¶

Xopt provides methods to parallelize optimizations using Processes, Threads, MPI, and Dask using the concurrent.futures interface as defined in https://www.python.org/dev/peps/pep-3148/ .

In [1]:

Copied!





# Helpers for this notebook
import multiprocessing
from concurrent.futures import ProcessPoolExecutor
from dask.distributed import Client

import matplotlib.pyplot as plt

import pandas as pd
from concurrent.futures import ThreadPoolExecutor

import os

from xopt import AsynchronousXopt as Xopt
from xopt.vocs import get_feasibility_data

SMOKE_TEST = os.environ.get("SMOKE_TEST")

# Notebook printing output
# from xopt import output_notebook
# output_notebook()

N_CPUS = multiprocessing.cpu_count()
N_CPUS

# directory for data.
os.makedirs("temp", exist_ok=True)
# Helpers for this notebook
import multiprocessing
from concurrent.futures import ProcessPoolExecutor
from dask.distributed import Client

import matplotlib.pyplot as plt

import pandas as pd
from concurrent.futures import ThreadPoolExecutor

import os

from xopt import AsynchronousXopt as Xopt
from xopt.vocs import get_feasibility_data

SMOKE_TEST = os.environ.get("SMOKE_TEST")

# Notebook printing output
# from xopt import output_notebook
# output_notebook()

N_CPUS = multiprocessing.cpu_count()
N_CPUS

# directory for data.
os.makedirs("temp", exist_ok=True)

The Xopt object can be instantiated from a JSON or YAML file, or a dict, with the proper structure.

Here we will make one

In [2]:

Copied!





# Make a proper input file.
MAX_EVALUATIONS = 32 if SMOKE_TEST else 1000
YAML = """

stopping_condition:
    name: MaxEvaluationsCondition
    max_evaluations: 1000

generator:
  name: cnsga
  output_path: temp
  population_size:  64
  vocs:
    variables:
      x1: [0, 3.14159]
      x2: [0, 3.14159]
    objectives: {y1: MINIMIZE, y2: MINIMIZE}
    constraints:
      c1: [GREATER_THAN, 0]
      c2: [LESS_THAN, 0.5]
    constants: {a: dummy_constant}
    
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    sleep: 0
    random_sleep: 0.1
  


"""
X = Xopt(YAML)
X.stopping_condition.max_evaluations = MAX_EVALUATIONS
X
# Make a proper input file.
MAX_EVALUATIONS = 32 if SMOKE_TEST else 1000
YAML = """

stopping_condition:
    name: MaxEvaluationsCondition
    max_evaluations: 1000

generator:
  name: cnsga
  output_path: temp
  population_size:  64
  vocs:
    variables:
      x1: [0, 3.14159]
      x2: [0, 3.14159]
    objectives: {y1: MINIMIZE, y2: MINIMIZE}
    constraints:
      c1: [GREATER_THAN, 0]
      c2: [LESS_THAN, 0.5]
    constants: {a: dummy_constant}
    
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    sleep: 0
    random_sleep: 0.1
  


"""
X = Xopt(YAML)
X.stopping_condition.max_evaluations = MAX_EVALUATIONS
X

Out[2]:

            Xopt
________________________________
Version: 0.1.dev1+gd4c488f75
Data size: 0
Config as YAML:
data_dump_file: null
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    raise_probability: 0
    random_sleep: 0.1
    sleep: 0
  max_workers: 1
  vectorized: false
generator:
  crossover_probability: 0.9
  mutation_probability: 1.0
  name: cnsga
  output_path: temp
  population: null
  population_file: null
  population_size: 64
  returns_id: false
  supports_constraints: true
  supports_multi_objective: true
  supports_single_objective: true
  vocs:
    constants:
      a:
        dtype: null
        type: Constant
        value: dummy_constant
    constraints:
      c1:
        dtype: null
        type: GreaterThanConstraint
        value: 0.0
      c2:
        dtype: null
        type: LessThanConstraint
        value: 0.5
    objectives:
      y1:
        dtype: null
        type: MinimizeObjective
      y2:
        dtype: null
        type: MinimizeObjective
    observables: {}
    variables:
      x1:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
      x2:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
is_done: false
serialize_inline: false
serialize_torch: false
stopping_condition:
  count_valid_only: false
  max_evaluations: 1000
  name: MaxEvaluationsCondition
  use_dataframe_index: false
strict: true
xopt_dump_file: null

In [3]:

Copied!

%%timeit
# Check that the average time is close to random_sleep
X.evaluator.function({"x1": 0.5, "x2": 0.5}, random_sleep=0.1)
%%timeit
# Check that the average time is close to random_sleep
X.evaluator.function({"x1": 0.5, "x2": 0.5}, random_sleep=0.1)

81.7 ms ± 14.5 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [4]:

Copied!

%%time
X.run()
%%time
X.run()

CPU times: user 6.43 s, sys: 37.4 ms, total: 6.47 s
Wall time: 1min 57s

Processes¶

In [5]:

Copied!





%%time
X = Xopt(YAML)

with ProcessPoolExecutor(max_workers=N_CPUS) as executor:
    X.evaluator.executor = executor
    X.evaluator.max_workers = N_CPUS
    X.run()
len(X.data)
%%time
X = Xopt(YAML)

with ProcessPoolExecutor(max_workers=N_CPUS) as executor:
    X.evaluator.executor = executor
    X.evaluator.max_workers = N_CPUS
    X.run()
len(X.data)

CPU times: user 5.9 s, sys: 177 ms, total: 6.08 s
Wall time: 29 s

Out[5]:

Threads¶

Continue running, this time with threads.

In [6]:

Copied!





%%time
X = Xopt(YAML)

with ThreadPoolExecutor(max_workers=N_CPUS) as executor:
    X.evaluator.executor = executor
    X.evaluator.max_workers = N_CPUS
    X.run()
len(X.data)
%%time
X = Xopt(YAML)

with ThreadPoolExecutor(max_workers=N_CPUS) as executor:
    X.evaluator.executor = executor
    X.evaluator.max_workers = N_CPUS
    X.run()
len(X.data)

CPU times: user 5.97 s, sys: 135 ms, total: 6.11 s
Wall time: 29 s

Out[6]:

MPI¶

The test.yaml file completely defines the problem. We will also direct the logging to an xopt.log file. The following invocation recruits 4 MPI workers to solve this problem.

We can also continue by calling .save with a JSON filename. This will write all of previous results into the file.

In [7]:

Copied!

X = Xopt(YAML)
X.dump("test.yaml")  # Write this input to file
!cat test.yaml
X = Xopt(YAML)
X.dump("test.yaml")  # Write this input to file
!cat test.yaml

data: null
data_dump_file: null
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    raise_probability: 0
    random_sleep: 0.1
    sleep: 0
  max_workers: 1
  vectorized: false
generator:
  crossover_probability: 0.9
  mutation_probability: 1.0
  name: cnsga
  output_path: temp
  population: null
  population_file: null
  population_size: 64
  returns_id: false
  supports_constraints: true
  supports_multi_objective: true
  supports_single_objective: true
  vocs:
    constants:
      a:
        dtype: null
        type: Constant
        value: dummy_constant
    constraints:
      c1:
        dtype: null
        type: GreaterThanConstraint
        value: 0.0
      c2:
        dtype: null
        type: LessThanConstraint
        value: 0.5
    objectives:
      y1:
        dtype: null
        type: MinimizeObjective
      y2:
        dtype: null
        type: MinimizeObjective
    observables: {}
    variables:
      x1:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
      x2:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
is_done: false
serialize_inline: false
serialize_torch: false
stopping_condition:
  count_valid_only: false
  max_evaluations: 1000
  name: MaxEvaluationsCondition
  use_dataframe_index: false
strict: true
xopt_dump_file: null

In [8]:

Copied!

%%time
!mpirun -n 8 python -m mpi4py.futures -m xopt.mpi.run -vv --logfile xopt.log test.yaml
%%time
!mpirun -n 8 python -m mpi4py.futures -m xopt.mpi.run -vv --logfile xopt.log test.yaml

Namespace(input_file='test.yaml', logfile='xopt.log', verbose=2, asynchronous=True)
Parallel execution with 8 workers
Enabling async mode

Initialized generator cnsga
Created toolbox with 2 variables, 2 constraints, and 2 objectives.
    Using selection algorithm: nsga2

            Xopt
________________________________
Version: 0.1.dev1+gd4c488f75
Data size: 0
Config as YAML:
data_dump_file: null
evaluator:
  function: xopt.resources.test_functions.tnk.evaluate_TNK
  function_kwargs:
    raise_probability: 0
    random_sleep: 0.1
    sleep: 0
  max_workers: 1
  vectorized: false
generator:
  crossover_probability: 0.9
  mutation_probability: 1.0
  name: cnsga
  output_path: temp
  population: null
  population_file: null
  population_size: 64
  returns_id: false
  supports_constraints: true
  supports_multi_objective: true
  supports_single_objective: true
  vocs:
    constants:
      a:
        dtype: null
        type: Constant
        value: dummy_constant
    constraints:
      c1:
        dtype: null
        type: GreaterThanConstraint
        value: 0.0
      c2:
        dtype: null
        type: LessThanConstraint
        value: 0.5
    objectives:
      y1:
        dtype: null
        type: MinimizeObjective
      y2:
        dtype: null
        type: MinimizeObjective
    observables: {}
    variables:
      x1:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
      x2:
        default_value: null
        domain:
        - 0.0
        - 3.14159
        dtype: null
        type: ContinuousVariable
is_done: false
serialize_inline: false
serialize_torch: false
stopping_condition:
  count_valid_only: false
  max_evaluations: 1000
  name: MaxEvaluationsCondition
  use_dataframe_index: false
strict: true
xopt_dump_file: null


Running Xopt

Xopt is done. Stopping condition met.

CPU times: user 307 ms, sys: 63.6 ms, total: 371 ms
Wall time: 31 s

In [9]:

Copied!

!tail xopt.log
!tail xopt.log

2026-09-03T16:39:55+0000 - xopt - INFO - Parallel execution with 8 workers
2026-09-03T16:39:55+0000 - xopt - INFO - Enabling async mode
2026-09-03T16:39:55+0000 - xopt.generator - INFO - Initialized generator cnsga
2026-09-03T16:39:55+0000 - xopt.generators.ga.cnsga - INFO - Created toolbox with 2 variables, 2 constraints, and 2 objectives.
2026-09-03T16:39:55+0000 - xopt.generators.ga.cnsga - INFO -     Using selection algorithm: nsga2
2026-09-03T16:39:55+0000 - xopt.base - INFO - Running Xopt
2026-09-03T16:40:19+0000 - xopt.base - INFO - Xopt is done. Stopping condition met.

Dask¶

In [10]:

Copied!

client = Client()
executor = client.get_executor()
client
client = Client()
executor = client.get_executor()
client

Out[10]:

Client

Client-27de8711-a7b6-11f1-8d89-e9ab551d2d27

Connection method: Cluster object	Cluster type: distributed.LocalCluster
Dashboard: http://127.0.0.1:8787/status

Cluster Info

LocalCluster

5c1c1477

Dashboard: http://127.0.0.1:8787/status	Workers: 4
Total threads: 4	Total memory: 15.61 GiB
Status: running	Using processes: True

Scheduler Info

Scheduler

Scheduler-a6283f07-b50e-4d3e-a416-2da360b646d1

Comm: tcp://127.0.0.1:35807	Workers: 0
Dashboard: http://127.0.0.1:8787/status	Total threads: 0
Started: Just now	Total memory: 0 B

Workers

Worker: 0

Comm: tcp://127.0.0.1:37349	Total threads: 1
Dashboard: http://127.0.0.1:36485/status	Memory: 3.90 GiB
Nanny: tcp://127.0.0.1:45371
Local directory: /tmp/dask-scratch-space/worker-1lymdpl3

Worker: 1

Comm: tcp://127.0.0.1:34601	Total threads: 1
Dashboard: http://127.0.0.1:37151/status	Memory: 3.90 GiB
Nanny: tcp://127.0.0.1:33995
Local directory: /tmp/dask-scratch-space/worker-fmix1tfh

Worker: 2

Comm: tcp://127.0.0.1:35117	Total threads: 1
Dashboard: http://127.0.0.1:34169/status	Memory: 3.90 GiB
Nanny: tcp://127.0.0.1:33885
Local directory: /tmp/dask-scratch-space/worker-bc8v3as4

Worker: 3

Comm: tcp://127.0.0.1:35373	Total threads: 1
Dashboard: http://127.0.0.1:37115/status	Memory: 3.90 GiB
Nanny: tcp://127.0.0.1:35645
Local directory: /tmp/dask-scratch-space/worker-r6euk10x

In [11]:

Copied!





%%time
X = Xopt(YAML)
X.evaluator.executor = executor
X.evaluator.max_workers = N_CPUS
X.run()
len(X.data)
%%time
X = Xopt(YAML)
X.evaluator.executor = executor
X.evaluator.max_workers = N_CPUS
X.run()
len(X.data)

CPU times: user 12.8 s, sys: 1.11 s, total: 13.9 s
Wall time: 35.8 s

Out[11]:

Load output into Pandas¶

This algorithm writes two types of files: gen_{i}.json with all of the new individuals evaluated in a generation, and pop_{i}.json with the latest best population. Xopt provides some functions to load these easily into a Pandas dataframe for further analysis.

In [12]:

Copied!

X.data
X.data

Out[12]:

	x1	x2	a	y1	y2	c1	c2	xopt_runtime	xopt_error
0	2.653590	0.530447	dummy_constant	2.653590	0.530447	6.422901	4.638876	0.199447	False
1	1.149788	2.982549	dummy_constant	1.149788	2.982549	9.125352	6.585276	0.193314	False
2	2.305700	1.488814	dummy_constant	2.305700	1.488814	6.629683	4.238306	0.133463	False
3	2.974469	2.307169	dummy_constant	2.974469	2.307169	13.213095	9.388859	0.009213	False
4	0.264828	0.039781	dummy_constant	0.264828	0.039781	-0.855527	0.267108	0.161016	False
...	...	...	...	...	...	...	...	...	...
995	1.000311	0.222020	dummy_constant	1.000311	0.222020	0.143749	0.327584	0.133725	False
996	0.948200	0.949789	dummy_constant	0.948200	0.949789	0.701193	0.403194	0.160495	False
997	0.612259	0.919386	dummy_constant	0.612259	0.919386	0.320100	0.188487	0.092607	False
998	0.075458	1.041649	dummy_constant	0.075458	1.041649	0.050520	0.473619	0.077740	False
999	0.722724	0.576435	dummy_constant	0.722724	0.576435	-0.123249	0.055448	0.034657	False

1000 rows × 9 columns

In [13]:

Copied!

df = pd.concat([X.data, get_feasibility_data(X.vocs, X.data)], axis=1)
df[df["feasible"]]
df = pd.concat([X.data, get_feasibility_data(X.vocs, X.data)], axis=1)
df[df["feasible"]]

Out[13]:

	x1	x2	a	y1	y2	c1	c2	xopt_runtime	xopt_error	feasible_c1	feasible_c2	feasible
7	0.580972	0.941368	dummy_constant	0.580972	0.941368	0.307471	0.201362	0.197148	False	True	True	True
16	1.012806	0.802907	dummy_constant	1.012806	0.802907	0.697170	0.354722	0.172281	False	True	True	True
22	1.004379	0.591954	dummy_constant	1.004379	0.591954	0.421059	0.262853	0.085070	False	True	True	True
23	0.989307	0.741547	dummy_constant	0.989307	0.741547	0.593351	0.297766	0.152439	False	True	True	True
74	0.796416	0.853737	dummy_constant	0.796416	0.853737	0.278185	0.212992	0.046908	False	True	True	True
...	...	...	...	...	...	...	...	...	...	...	...	...
994	1.059137	0.111603	dummy_constant	1.059137	0.111603	0.145100	0.463487	0.001662	False	True	True	True
995	1.000311	0.222020	dummy_constant	1.000311	0.222020	0.143749	0.327584	0.133725	False	True	True	True
996	0.948200	0.949789	dummy_constant	0.948200	0.949789	0.701193	0.403194	0.160495	False	True	True	True
997	0.612259	0.919386	dummy_constant	0.612259	0.919386	0.320100	0.188487	0.092607	False	True	True	True
998	0.075458	1.041649	dummy_constant	0.075458	1.041649	0.050520	0.473619	0.077740	False	True	True	True

482 rows × 12 columns

In [14]:

Copied!

# Plot the feasible ones
feasible_df = df[df["feasible"]]
feasible_df.plot("y1", "y2", kind="scatter").set_aspect("equal")
# Plot the feasible ones
feasible_df = df[df["feasible"]]
feasible_df.plot("y1", "y2", kind="scatter").set_aspect("equal")

No description has been provided for this image

In [15]:

Copied!

# Plot the infeasible ones
infeasible_df = df[~df["feasible"]]
infeasible_df.plot("y1", "y2", kind="scatter").set_aspect("equal")
# Plot the infeasible ones
infeasible_df = df[~df["feasible"]]
infeasible_df.plot("y1", "y2", kind="scatter").set_aspect("equal")

In [16]:

Copied!

# This is the final population
df1 = X.generator.population
df1.plot("y1", "y2", kind="scatter").set_aspect("equal")
# This is the final population
df1 = X.generator.population
df1.plot("y1", "y2", kind="scatter").set_aspect("equal")

matplotlib plotting¶

You can always use matplotlib for customizable plotting

In [17]:

Copied!





# Extract objectives from output
k1, k2 = "y1", "y2"

fig, ax = plt.subplots(figsize=(6, 6))

ax.scatter(
    infeasible_df[k1],
    infeasible_df[k2],
    color="blue",
    marker=".",
    alpha=0.5,
    label="infeasible",
)
ax.scatter(
    feasible_df[k1], feasible_df[k2], color="orange", marker=".", label="feasible"
)
ax.scatter(df1[k1], df1[k2], color="red", marker=".", label="final population")
ax.set_xlabel(k1)
ax.set_ylabel(k2)
ax.set_aspect("auto")
ax.set_title("Xopt's CNSGA algorithm")
plt.legend()
# Extract objectives from output
k1, k2 = "y1", "y2"

fig, ax = plt.subplots(figsize=(6, 6))

ax.scatter(
    infeasible_df[k1],
    infeasible_df[k2],
    color="blue",
    marker=".",
    alpha=0.5,
    label="infeasible",
)
ax.scatter(
    feasible_df[k1], feasible_df[k2], color="orange", marker=".", label="feasible"
)
ax.scatter(df1[k1], df1[k2], color="red", marker=".", label="final population")
ax.set_xlabel(k1)
ax.set_ylabel(k2)
ax.set_aspect("auto")
ax.set_title("Xopt's CNSGA algorithm")
plt.legend()

Out[17]:

<matplotlib.legend.Legend at 0x7f3418c3dc70>

In [18]:

Copied!





# Cleanup
#!rm -r dask-worker-space
!rm -r temp
!rm xopt.log*
!rm test.yaml
# Cleanup
#!rm -r dask-worker-space
!rm -r temp
!rm xopt.log*
!rm test.yaml