Seed rotations

* Seeds can now be specified at the benchmark level and rotated (the first seed
  will be used for the first run of every algorithm-problem pair etc.)
* `WrappedProblem`s can now be reseeded.

Related to #7

Author: altaris

CHANGELOG.md

@@ -5,6 +5,10 @@ Changelog
 
 ## New features
 
+* Seed rotations: `Benchmark.__init__` now has a `seeds` argument which can
+  receive a list of seeds. The first seed will be used for all random
+  generators involved in the first run of every algorithm-problem pair, the
+  second for all second runs, etc.
 * When constructing a `WrappedProblem`, the wrapped problem is now deepcopyied
   by default:
   ```py
@@ -35,6 +39,9 @@ Changelog
 
 ## Breaking changes
 
+* Seeds can no longer be specified in algorithm description dicts (see
+  `Benchmark.__init__`). Instead, use the `seeds` argument when constructing
+  benchmarks (see above).
 * Class `nmoo.benchmark.Pair` has been replaced by `nmoo.benchmark.PAPair`,
   representing a problem-algorithm pair, and `nmoo.benchmark.PARTriple`,
   representing a problem-algorithm-(run number) triple. Method

example.py

@@ -56,4 +56,5 @@ def make_benchmark() -> nmoo.Benchmark:
             },
         },
         n_runs=3,
+        seeds=[123, 456, 789],
     )

nmoo/benchmark.py

@@ -145,6 +145,11 @@ class Benchmark:
     Results of all runs.
     """
 
+    _seeds: List[Optional[int]]
+    """
+    List of seeds to use. Must be of length `_n_runs`.
+    """
+
     def __init__(
         self,
         output_dir_path: Union[Path, str],
@@ -154,10 +159,11 @@ def __init__(
         dump_histories: bool = True,
         performance_indicators: Optional[List[str]] = None,
         max_retry: int = -1,
+        seeds: Optional[List[Optional[int]]] = None,
     ):
         """
         Constructor. The set of problems to be benchmarked is represented by a
-        dictionary with the following structure::
+        dictionary with the following structure:
 
             problems = {
                 <problem_name>: <problem_description>,
@@ -198,9 +204,8 @@ def __init__(
         * `evaluator` (optional): an algorithm evaluator object; note that it
             is deepcopied for every run of `minimize`;
         * `return_least_infeasible` (optional, bool): if the algorithm cannot
-            find a feasable solution, wether the least infeasable solution
+            find a feasible solution, wether the least infeasible solution
             should still be returned; defaults to `False`;
-        * `seed` (optional, int): a seed;
         * `termination` (optional): a pymoo termination criterion; note that it
             is deepcopied for every run of `minimize`;
         * `verbose` (optional, bool): wether outputs should be printed during
@@ -250,6 +255,9 @@ def __init__(
                 In the result dataframe, the corresponding columns will be
                 named `perf_<name of indicator>`, e.g. `perf_igd`. If left
                 unspecified, defaults to `["igd"]`.
+            seeds (Optional[List[Optional[int]]]): List of seeds to use. The
+                first seed will be used for the first run of every
+                algorithm-problem pair, etc.
         """
         self._output_dir_path = Path(output_dir_path)
         self._set_problems(problems)
@@ -263,6 +271,12 @@ def __init__(
         self._dump_histories = dump_histories
         self._set_performance_indicators(performance_indicators)
         self._max_retry = max_retry
+        if seeds is None:
+            self._seeds = [None] * n_runs
+        elif len(seeds) != n_runs:
+            raise ValueError("Seed list must be of length n_runs")
+        else:
+            self._seeds = seeds
 
     def _compute_global_pareto_population(self, pair: PAPair) -> None:
         """
@@ -457,8 +471,11 @@ def _run_par_triple(
             triple.algorithm_description.get("evaluator"),
         )
         try:
+            seed = self._seeds[triple.n_run - 1]
+            problem = deepcopy(triple.problem_description["problem"])
+            problem.reseed(seed)
             results = minimize(
-                deepcopy(triple.problem_description["problem"]),
+                problem,
                 triple.algorithm_description["algorithm"],
                 termination=triple.algorithm_description.get("termination"),
                 copy_algorithm=True,
@@ -471,7 +488,7 @@ def _run_par_triple(
                     "return_least_infeasible", False
                 ),
                 save_history=True,
-                seed=triple.algorithm_description.get("seed"),
+                seed=seed,
                 verbose=triple.algorithm_description.get("verbose", False),
             )
         except Exception as e:  # pylint: disable=broad-except

nmoo/noises/gaussian.py

@@ -100,7 +100,7 @@ def __init__(
                 "Invalid noise specification. Either mean and covariance are "
                 "both set, or a parameters dict is set."
             )
-        self._generator = np.random.default_rng(seed)
+        self.reseed(seed)
 
     def _evaluate(self, x, out, *args, **kwargs):
         """
@@ -147,3 +147,8 @@ def _evaluate(self, x, out, *args, **kwargs):
         self.add_to_history_x_out(
             x, out, **{k + "_noise": v for k, v in noises.items()}
         )
+
+    def reseed(self, seed: Any) -> None:
+        self._generator = np.random.default_rng(seed)
+        if isinstance(self._problem, WrappedProblem):
+            self._problem.reseed(seed)

nmoo/noises/uniform.py

@@ -118,7 +118,7 @@ def __init__(
                         self._parameters[k].append((-a, a))
         except AssertionError as e:
             raise ValueError("Invalid noise parameters") from e
-        self._generator = np.random.default_rng(seed)
+        self.reseed(seed)
 
     # pylint: disable=duplicate-code
     def _evaluate(self, x, out, *args, **kwargs):
@@ -146,3 +146,8 @@ def _evaluate(self, x, out, *args, **kwargs):
         self.add_to_history_x_out(
             x, out, **{k + "_noise": v for k, v in noises.items()}
         )
+
+    def reseed(self, seed: Any) -> None:
+        self._generator = np.random.default_rng(seed)
+        if isinstance(self._problem, WrappedProblem):
+            self._problem.reseed(seed)

nmoo/wrapped_problem.py

@@ -15,7 +15,7 @@
 import logging
 from copy import deepcopy
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Any, Dict, List, Union
 
 import numpy as np
 from pymoo.core.problem import Problem
@@ -202,6 +202,25 @@ def dump_history(self, path: Union[Path, str], compressed: bool = True):
         saver = np.savez_compressed if compressed else np.savez
         saver(path, **self._history)
 
+    def ground_problem(self) -> Problem:
+        """
+        Recursively goes down the problem wrappers until an actual
+        `pymoo.Problem` is found, and returns it.
+        """
+        if isinstance(self._problem, WrappedProblem):
+            return self._problem.ground_problem()
+        return self._problem
+
+    def reseed(self, seed: Any) -> None:
+        """
+        Recursively resets the internal random state of the problem. See the
+        [numpy
+        documentation](https://numpy.org/doc/stable/reference/random/generator.html?highlight=default_rng#numpy.random.default_rng)
+        for details about acceptable seeds.
+        """
+        if isinstance(self._problem, WrappedProblem):
+            self._problem.reseed(seed)
+
     def start_new_run(self):
         """
         In short, it rotates the history of the current problem, and all
@@ -220,15 +239,6 @@ def start_new_run(self):
         if isinstance(self._problem, WrappedProblem):
             self._problem.start_new_run()
 
-    def ground_problem(self) -> Problem:
-        """
-        Recursively goes down the problem wrappers until an actual
-        `pymoo.Problem` is found, and returns it.
-        """
-        if isinstance(self._problem, WrappedProblem):
-            return self._problem.ground_problem()
-        return self._problem
-
     def _evaluate(self, x, out, *args, **kwargs):
         """
         Calls the wrapped problems's `_evaluate` method and appends its input