chore: fail fast when score corruption detected

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/report/BenchmarkReport.java

@@ -316,10 +316,11 @@ public List<String> getWarningList() {
                     + " Maybe reduce the parallelBenchmarkCount.");
         }
         EnvironmentMode environmentMode = plannerBenchmarkResult.getEnvironmentMode();
-        if (environmentMode != null && environmentMode.isAsserted()) {
-            warningList.add("The environmentMode (" + environmentMode + ") is asserting."
-                    + " This decreases performance."
-                    + " Maybe set the environmentMode to " + EnvironmentMode.REPRODUCIBLE + ".");
+        if (environmentMode != null && environmentMode.isStepAssertOrMore()) {
+            // Phase assert performance impact is negligible.
+            warningList.add(
+                    "The environmentMode (%s) is step-asserting or more. This decreases performance. Maybe set the environmentMode to %s."
+                            .formatted(environmentMode, EnvironmentMode.PHASE_ASSERT));
         }
         LoggingLevel loggingLevelTimefoldCore = plannerBenchmarkResult.getLoggingLevelTimefoldSolverCore();
         if (loggingLevelTimefoldCore == LoggingLevel.TRACE) {

benchmark/src/main/resources/benchmark.xsd

@@ -2630,9 +2630,18 @@
       <xs:enumeration value="FAST_ASSERT"/>
 
 
+      <xs:enumeration value="STEP_ASSERT"/>
+
+
+      <xs:enumeration value="PHASE_ASSERT"/>
+
+
       <xs:enumeration value="REPRODUCIBLE"/>
 
 
+      <xs:enumeration value="NO_ASSERT"/>
+
+
       <xs:enumeration value="NON_REPRODUCIBLE"/>
 
 

core/src/main/java/ai/timefold/solver/core/config/solver/EnvironmentMode.java

@@ -1,9 +1,13 @@
 package ai.timefold.solver.core.config.solver;
 
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Random;
 
 import jakarta.xml.bind.annotation.XmlEnum;
 
+import ai.timefold.solver.core.api.domain.entity.PlanningEntity;
+import ai.timefold.solver.core.api.domain.variable.VariableListener;
 import ai.timefold.solver.core.api.solver.Solver;
 import ai.timefold.solver.core.impl.heuristic.move.Move;
 import ai.timefold.solver.core.impl.score.director.InnerScoreDirector;
@@ -25,23 +29,22 @@ public enum EnvironmentMode {
      * a constraint, the engine itself or something else at the highest performance cost.
      * <p>
      * Because it tracks genuine and shadow variables, it is able to report precisely what variables caused the corruption and
-     * report any missed {@link ai.timefold.solver.core.api.domain.variable.VariableListener} events.
+     * report any missed {@link VariableListener} events.
      * <p>
-     * This mode is reproducible (see {@link #REPRODUCIBLE} mode).
+     * This mode is reproducible (see {@link #PHASE_ASSERT} mode).
      * <p>
      * This mode is intrusive because it calls the {@link InnerScoreDirector#calculateScore()} more frequently than a non assert
      * mode.
      * <p>
      * This mode is by far the slowest of all the modes.
      */
     TRACKED_FULL_ASSERT(true),
-
     /**
      * This mode turns on all assertions
      * to fail-fast on a bug in a {@link Move} implementation, a constraint, the engine itself or something else
      * at a horrible performance cost.
      * <p>
-     * This mode is reproducible (see {@link #REPRODUCIBLE} mode).
+     * This mode is reproducible (see {@link #PHASE_ASSERT} mode).
      * <p>
      * This mode is intrusive because it calls the {@link InnerScoreDirector#calculateScore()} more frequently
      * than a non assert mode.
@@ -54,49 +57,71 @@ public enum EnvironmentMode {
      * to fail-fast on a bug in a {@link Move} implementation, a constraint, the engine itself or something else
      * at an overwhelming performance cost.
      * <p>
-     * This mode is reproducible (see {@link #REPRODUCIBLE} mode).
+     * This mode is reproducible (see {@link #PHASE_ASSERT} mode).
      * <p>
-     * This mode is non-intrusive, unlike {@link #FULL_ASSERT} and {@link #FAST_ASSERT}.
+     * This mode is non-intrusive, unlike {@link #FULL_ASSERT} and {@link #STEP_ASSERT}.
      * <p>
      * This mode is horribly slow.
      */
     NON_INTRUSIVE_FULL_ASSERT(true),
     /**
-     * This mode turns on several assertions (but not all of them)
-     * to fail-fast on a bug in a {@link Move} implementation, a constraint rule, the engine itself or something else
+     * @deprecated Prefer {@link #STEP_ASSERT}.
+     */
+    @Deprecated(forRemoval = true, since = "1.20.0")
+    FAST_ASSERT(true),
+    /**
+     * This mode turns on several assertions to fail-fast
+     * on a bug in a {@link Move} implementation, a constraint rule, the engine itself or something else
      * at a reasonable performance cost (in development at least).
      * <p>
-     * This mode is reproducible (see {@link #REPRODUCIBLE} mode).
+     * This mode is reproducible (see {@link #PHASE_ASSERT} mode).
      * <p>
      * This mode is intrusive because it calls the {@link InnerScoreDirector#calculateScore()} more frequently
-     * than a non assert mode.
+     * than a non-assert mode.
      * <p>
      * This mode is slow.
      */
-    FAST_ASSERT(true),
+    STEP_ASSERT(true),
     /**
-     * The reproducible mode is the default mode because it is recommended during development.
-     * In this mode, 2 runs on the same computer will execute the same code in the same order.
+     * This is the default mode as it is recommended during development,
+     * and runs minimal correctness checks that serve to quickly identify score corruption bugs.
+     * <p>
+     * In this mode, two runs on the same computer will execute the same code in the same order.
      * They will also yield the same result, except if they use a time based termination
      * and they have a sufficiently large difference in allocated CPU time.
      * This allows you to benchmark new optimizations (such as a new {@link Move} implementation) fairly
      * and reproduce bugs in your code reliably.
      * <p>
-     * Warning: some code can disrupt reproducibility regardless of this mode. See the reference manual for more info.
+     * Warning: some code can disrupt reproducibility regardless of this mode.
+     * This typically happens when user code serves data such as {@link PlanningEntity planning entities}
+     * from collections without defined iteration order, such as {@link HashSet} or {@link HashMap}.
      * <p>
      * In practice, this mode uses the default random seed,
-     * and it also disables certain concurrency optimizations (such as work stealing).
+     * and it also disables certain concurrency optimizations, such as work stealing.
      */
+    PHASE_ASSERT(true),
+    /**
+     * @deprecated Prefer {@link #NO_ASSERT}.
+     */
+    @Deprecated(forRemoval = true, since = "1.20.0")
     REPRODUCIBLE(false),
     /**
-     * The non reproducible mode is equally fast or slightly faster than {@link #REPRODUCIBLE}.
+     * As defined by {@link #PHASE_ASSERT}, but disables every single bug detection mechanism.
+     * This mode will run negligibly faster than {@link #PHASE_ASSERT},
+     * but will allow some bugs in user code (such as score corruptions) to go unnoticed.
+     * Use this mode when you are confident that your code is bug-free,
+     * or when you want to ignore a known bug temporarily.
+     */
+    NO_ASSERT(false),
+    /**
+     * The non-reproducible mode is equally fast or slightly faster than {@link #NO_ASSERT}.
      * <p>
      * The random seed is different on every run, which makes it more robust against an unlucky random seed.
      * An unlucky random seed gives a bad result on a certain data set with a certain solver configuration.
-     * Note that in most use cases the impact of the random seed is relatively low on the result.
+     * Note that in most use cases, the impact of the random seed is relatively low on the result.
      * An occasional bad result is far more likely to be caused by another issue (such as a score trap).
      * <p>
-     * In multithreaded scenarios, this mode allows the use of work stealing and other non deterministic speed tricks.
+     * In multithreaded scenarios, this mode allows the use of work stealing and other non-deterministic speed tricks.
      */
     NON_REPRODUCIBLE(false);
 
@@ -106,22 +131,47 @@ public enum EnvironmentMode {
         this.asserted = asserted;
     }
 
+    public boolean isStepAssertOrMore() {
+        if (!isAsserted()) {
+            return false;
+        }
+        return this != PHASE_ASSERT;
+    }
+
     public boolean isAsserted() {
         return asserted;
     }
 
+    public boolean isFullyAsserted() {
+        return switch (this) {
+            case TRACKED_FULL_ASSERT, FULL_ASSERT, NON_INTRUSIVE_FULL_ASSERT -> true;
+            case STEP_ASSERT, FAST_ASSERT, PHASE_ASSERT, REPRODUCIBLE, NO_ASSERT, NON_REPRODUCIBLE -> false;
+        };
+    }
+
+    /**
+     * @deprecated Use {@link #isFullyAsserted()} instead.
+     */
+    @Deprecated(forRemoval = true, since = "1.20.0")
     public boolean isNonIntrusiveFullAsserted() {
-        if (!isAsserted()) {
-            return false;
-        }
-        return this != FAST_ASSERT;
+        return isFullyAsserted();
     }
 
+    public boolean isIntrusivelyAsserted() {
+        return switch (this) {
+            // STEP_ASSERT = former FAST_ASSERT
+            case TRACKED_FULL_ASSERT, FULL_ASSERT, STEP_ASSERT, FAST_ASSERT -> true;
+            // NO_ASSERT = former REPRODUCIBLE
+            case NON_INTRUSIVE_FULL_ASSERT, PHASE_ASSERT, NO_ASSERT, NON_REPRODUCIBLE, REPRODUCIBLE -> false;
+        };
+    }
+
+    /**
+     * @deprecated Use {@link #isIntrusivelyAsserted()} instead.
+     */
+    @Deprecated(forRemoval = true, since = "1.20.0")
     public boolean isIntrusiveFastAsserted() {
-        if (!isAsserted()) {
-            return false;
-        }
-        return this != NON_INTRUSIVE_FULL_ASSERT;
+        return isIntrusivelyAsserted();
     }
 
     public boolean isReproducible() {
@@ -131,4 +181,5 @@ public boolean isReproducible() {
     public boolean isTracking() {
         return this == TRACKED_FULL_ASSERT;
     }
+
 }

core/src/main/java/ai/timefold/solver/core/config/solver/SolverConfig.java

@@ -633,7 +633,7 @@ public boolean canTerminate() {
     }
 
     public @NonNull EnvironmentMode determineEnvironmentMode() {
-        return Objects.requireNonNullElse(environmentMode, EnvironmentMode.REPRODUCIBLE);
+        return Objects.requireNonNullElse(environmentMode, EnvironmentMode.PHASE_ASSERT);
     }
 
     public @NonNull DomainAccessType determineDomainAccessType() {

core/src/main/java/ai/timefold/solver/core/impl/bavet/common/AbstractGroupNode.java

@@ -81,7 +81,7 @@ protected AbstractGroupNode(int groupStoreIndex, int undoStoreIndex,
                         updateOutTupleToFinisher(outTuple, group.getResultContainer());
                     }
                 }) : new DynamicPropagationQueue<>(nextNodesTupleLifecycle);
-        this.useAssertingGroupKey = environmentMode.isAsserted();
+        this.useAssertingGroupKey = environmentMode.isStepAssertOrMore();
     }
 
     protected AbstractGroupNode(int groupStoreIndex,

core/src/main/java/ai/timefold/solver/core/impl/constructionheuristic/DefaultConstructionHeuristicPhase.java

@@ -1,6 +1,7 @@
 package ai.timefold.solver.core.impl.constructionheuristic;
 
 import ai.timefold.solver.core.api.domain.solution.PlanningSolution;
+import ai.timefold.solver.core.config.solver.EnvironmentMode;
 import ai.timefold.solver.core.impl.constructionheuristic.decider.ConstructionHeuristicDecider;
 import ai.timefold.solver.core.impl.constructionheuristic.placer.EntityPlacer;
 import ai.timefold.solver.core.impl.constructionheuristic.scope.ConstructionHeuristicPhaseScope;
@@ -220,6 +221,12 @@ public DefaultConstructionHeuristicPhaseBuilder(int phaseIndex, boolean lastInit
             this.decider = decider;
         }
 
+        @Override
+        public DefaultConstructionHeuristicPhaseBuilder<Solution_> enableAssertions(EnvironmentMode environmentMode) {
+            super.enableAssertions(environmentMode);
+            return this;
+        }
+
         public EntityPlacer<Solution_> getEntityPlacer() {
             return entityPlacer;
         }

core/src/main/java/ai/timefold/solver/core/impl/constructionheuristic/DefaultConstructionHeuristicPhaseFactory.java

@@ -69,18 +69,10 @@ protected DefaultConstructionHeuristicPhaseBuilder<Solution_> createBuilder(
             HeuristicConfigPolicy<Solution_> phaseConfigPolicy, Termination<Solution_> solverTermination, int phaseIndex,
             boolean lastInitializingPhase, EntityPlacer<Solution_> entityPlacer) {
         var phaseTermination = buildPhaseTermination(phaseConfigPolicy, solverTermination);
-        var builder = new DefaultConstructionHeuristicPhaseBuilder<>(phaseIndex, lastInitializingPhase,
+        return new DefaultConstructionHeuristicPhaseBuilder<>(phaseIndex, lastInitializingPhase,
                 phaseConfigPolicy.getLogIndentation(), phaseTermination, entityPlacer,
-                buildDecider(phaseConfigPolicy, phaseTermination));
-        var environmentMode = phaseConfigPolicy.getEnvironmentMode();
-        if (environmentMode.isNonIntrusiveFullAsserted()) {
-            builder.setAssertStepScoreFromScratch(true);
-        }
-        if (environmentMode.isIntrusiveFastAsserted()) {
-            builder.setAssertExpectedStepScore(true);
-            builder.setAssertShadowVariablesAreNotStaleAfterStep(true);
-        }
-        return builder;
+                buildDecider(phaseConfigPolicy, phaseTermination))
+                .enableAssertions(phaseConfigPolicy.getEnvironmentMode());
     }
 
     @Override

core/src/main/java/ai/timefold/solver/core/impl/constructionheuristic/decider/ConstructionHeuristicDecider.java

@@ -50,8 +50,8 @@ public ConstructionHeuristicForager<Solution_> getForager() {
     }
 
     public void enableAssertions(EnvironmentMode environmentMode) {
-        this.assertMoveScoreFromScratch = environmentMode.isNonIntrusiveFullAsserted();
-        this.assertExpectedUndoMoveScore = environmentMode.isIntrusiveFastAsserted();
+        this.assertMoveScoreFromScratch = environmentMode.isFullyAsserted();
+        this.assertExpectedUndoMoveScore = environmentMode.isIntrusivelyAsserted();
     }
 
     // ************************************************************************

core/src/main/java/ai/timefold/solver/core/impl/exhaustivesearch/DefaultExhaustiveSearchPhase.java

@@ -9,6 +9,7 @@
 
 import ai.timefold.solver.core.api.domain.solution.PlanningSolution;
 import ai.timefold.solver.core.api.score.Score;
+import ai.timefold.solver.core.config.solver.EnvironmentMode;
 import ai.timefold.solver.core.impl.exhaustivesearch.decider.ExhaustiveSearchDecider;
 import ai.timefold.solver.core.impl.exhaustivesearch.node.ExhaustiveSearchLayer;
 import ai.timefold.solver.core.impl.exhaustivesearch.node.ExhaustiveSearchNode;
@@ -244,12 +245,12 @@ public Builder(int phaseIndex, String logIndentation, Termination<Solution_> pha
             this.decider = decider;
         }
 
-        public void setAssertWorkingSolutionScoreFromScratch(boolean assertWorkingSolutionScoreFromScratch) {
-            this.assertWorkingSolutionScoreFromScratch = assertWorkingSolutionScoreFromScratch;
-        }
-
-        public void setAssertExpectedWorkingSolutionScore(boolean assertExpectedWorkingSolutionScore) {
-            this.assertExpectedWorkingSolutionScore = assertExpectedWorkingSolutionScore;
+        @Override
+        public Builder<Solution_> enableAssertions(EnvironmentMode environmentMode) {
+            super.enableAssertions(environmentMode);
+            assertWorkingSolutionScoreFromScratch = environmentMode.isFullyAsserted();
+            assertExpectedWorkingSolutionScore = environmentMode.isIntrusivelyAsserted();
+            return this;
         }
 
         @Override

core/src/main/java/ai/timefold/solver/core/impl/exhaustivesearch/DefaultExhaustiveSearchPhaseFactory.java

@@ -83,22 +83,12 @@ public ExhaustiveSearchPhase<Solution_> buildPhase(int phaseIndex, boolean lastI
                 EntitySelectorFactory.<Solution_> create(entitySelectorConfig_)
                         .buildEntitySelector(phaseConfigPolicy, SelectionCacheType.PHASE, SelectionOrder.ORIGINAL);
 
-        DefaultExhaustiveSearchPhase.Builder<Solution_> builder = new DefaultExhaustiveSearchPhase.Builder<>(phaseIndex,
+        return new DefaultExhaustiveSearchPhase.Builder<>(phaseIndex,
                 solverConfigPolicy.getLogIndentation(), phaseTermination,
                 nodeExplorationType_.buildNodeComparator(scoreBounderEnabled), entitySelector, buildDecider(phaseConfigPolicy,
-                        entitySelector, bestSolutionRecaller, phaseTermination, scoreBounderEnabled));
-
-        EnvironmentMode environmentMode = phaseConfigPolicy.getEnvironmentMode();
-        if (environmentMode.isNonIntrusiveFullAsserted()) {
-            builder.setAssertWorkingSolutionScoreFromScratch(true);
-            builder.setAssertStepScoreFromScratch(true); // Does nothing because ES doesn't use predictStepScore()
-        }
-        if (environmentMode.isIntrusiveFastAsserted()) {
-            builder.setAssertExpectedWorkingSolutionScore(true);
-            builder.setAssertExpectedStepScore(true); // Does nothing because ES doesn't use predictStepScore()
-            builder.setAssertShadowVariablesAreNotStaleAfterStep(true); // Does nothing because ES doesn't use predictStepScore()
-        }
-        return builder.build();
+                        entitySelector, bestSolutionRecaller, phaseTermination, scoreBounderEnabled))
+                .enableAssertions(phaseConfigPolicy.getEnvironmentMode())
+                .build();
     }
 
     private EntitySelectorConfig buildEntitySelectorConfig(HeuristicConfigPolicy<Solution_> configPolicy) {
@@ -150,13 +140,12 @@ private ExhaustiveSearchDecider<Solution_> buildDecider(HeuristicConfigPolicy<So
                 ? new TrendBasedScoreBounder(configPolicy.getScoreDefinition(), configPolicy.getInitializingScoreTrend())
                 : null;
         ExhaustiveSearchDecider<Solution_> decider = new ExhaustiveSearchDecider<>(configPolicy.getLogIndentation(),
-                bestSolutionRecaller, termination,
-                manualEntityMimicRecorder, moveSelector, scoreBounderEnabled, scoreBounder);
+                bestSolutionRecaller, termination, manualEntityMimicRecorder, moveSelector, scoreBounderEnabled, scoreBounder);
         EnvironmentMode environmentMode = configPolicy.getEnvironmentMode();
-        if (environmentMode.isNonIntrusiveFullAsserted()) {
+        if (environmentMode.isFullyAsserted()) {
             decider.setAssertMoveScoreFromScratch(true);
         }
-        if (environmentMode.isIntrusiveFastAsserted()) {
+        if (environmentMode.isIntrusivelyAsserted()) {
             decider.setAssertExpectedUndoMoveScore(true);
         }
         return decider;

core/src/main/java/ai/timefold/solver/core/impl/heuristic/selector/move/factory/MoveIteratorFactory.java

@@ -65,7 +65,7 @@ default void phaseEnded(ScoreDirector<Solution_> scoreDirector) {
      * @param scoreDirector never null, the {@link ScoreDirector}
      *        which has the {@link ScoreDirector#getWorkingSolution()} of which the {@link Move}s need to be generated
      * @param workingRandom never null, the {@link Random} to use when any random number is needed,
-     *        so {@link EnvironmentMode#REPRODUCIBLE} works correctly
+     *        so {@link EnvironmentMode#PHASE_ASSERT} works correctly
      * @return never null, an {@link Iterator} that is allowed (or even presumed) to be never ending
      * @throws UnsupportedOperationException if only {@link #createOriginalMoveIterator(ScoreDirector)} is supported
      */

core/src/main/java/ai/timefold/solver/core/impl/localsearch/DefaultLocalSearchPhase.java

@@ -8,6 +8,7 @@
 
 import ai.timefold.solver.core.api.domain.solution.PlanningSolution;
 import ai.timefold.solver.core.api.score.constraint.ConstraintMatchTotal;
+import ai.timefold.solver.core.config.solver.EnvironmentMode;
 import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
 import ai.timefold.solver.core.impl.localsearch.decider.LocalSearchDecider;
 import ai.timefold.solver.core.impl.localsearch.event.LocalSearchPhaseLifecycleListener;
@@ -235,6 +236,12 @@ public Builder(int phaseIndex, String logIndentation, Termination<Solution_> pha
             this.decider = decider;
         }
 
+        @Override
+        public Builder<Solution_> enableAssertions(EnvironmentMode environmentMode) {
+            super.enableAssertions(environmentMode);
+            return this;
+        }
+
         @Override
         public DefaultLocalSearchPhase<Solution_> build() {
             return new DefaultLocalSearchPhase<>(this);