Add ecm_late.ep_ (a 25-position slow-to-confirm ECM subset) and wire run_tests.sh to run it sn-only, as a search/extensions-quality metric distinct from the curated suites' solve-count gate.

Positions are ECM problems where the correct move only becomes -- and stays -- the reported best move well into the search (5+ seconds, confirmed stable for 2+ further iterations, non-zero final score) across a majority of 8 independent full-suite runs; derived from mining 8 full 879-position ECM runs at 20s/move for exactly this pattern, then filtering out the "engine got lucky on the last iteration" and "drawn by repetition" false positives that a naive late-solve detector picks up (cross-checked against tests/ECM.md's existing flaky-position list).

sn-only (no sd sweep) because a fixed depth on 1 thread, with no cutoff-sharing across threads to shrink the tree, can blow up unpredictably on the sharpest of these positions; a fixed 15M-node budget keeps wall-clock bounded (~5min/suite) while staying in the right ballpark of the per-thread node share implied by the 8-thread discovery data.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

1 files changed, 57 insertions, 20 deletions

diff --git a/src/run_tests.sh b/src/run_tests.sh
index 28e8ffe..8f2d3f1 100755
--- a/src/run_tests.sh
+++ b/src/run_tests.sh
@@ -17,31 +17,64 @@ fi
 # sd (fixed depth) is the right choice when judging whether a pruning/
 # ordering change makes the tree smaller or larger for the same search
 # effort; sn (fixed node budget) is the right choice when judging how many
-# positions solve within a fixed cost. Override via SD/SN env vars.
+# positions solve within a fixed cost. Override via SD/SN env vars --
+# this applies to every suite passed on the command line.
+#
+# ecm_late.ep_ (see its header comment) is a different shape of suite --
+# curated for positions where the *correct* move is only confirmed well
+# into the search, specifically so avg-solution-time/total-nodecount from
+# the stats block below can be read as an extensions/search-quality
+# metric ("does the engine confirm these tactics sooner, in aggregate")
+# rather than a solve-count gate. It's sn-only (no sd sweep): the
+# discovery data behind it was measured on an 8-thread box (avg ~13 ply /
+# ~70M total nodes across all 8 threads) and a fixed *depth* on 1 thread,
+# with no cutoff-sharing across threads to shrink the tree, can blow up
+# unpredictably on the sharpest of these positions -- a fixed *node*
+# budget bounds wall-clock regardless. 15M nodes (~12s/position at this
+# box's ~1.3Mnps single-thread rate, ~5 min for all 25 positions) is
+# picked to stay in "quick ad hoc iteration" territory while still being
+# in the right ballpark of the per-thread node share implied by the
+# 8-thread discovery data -- not a rigorously derived number, adjust via
+# SN if it's cutting off too many positions before they confirm.
 SD="${SD:-10}"
 SN="${SN:-5000000}"
 SN_LABEL="${SN_LABEL:-5m}"
+ECM_LATE_SN=15000000
+ECM_LATE_SN_LABEL=15m
+
+suite_sn_only() {
+    [ "$1" = "ecm_late" ]
+}
+suite_sn() {
+    if [ "$1" = "ecm_late" ]; then echo "$ECM_LATE_SN"; else echo "$SN"; fi
+}
+suite_sn_label() {
+    if [ "$1" = "ecm_late" ]; then echo "$ECM_LATE_SN_LABEL"; else echo "$SN_LABEL"; fi
+}
 
 /bin/rm -rf "${LOGDIR}"
 mkdir -p "${LOGDIR}"
 
-echo -n "Launching test suites (${SUITES[*]}) @ sn=${SN} and sd=${SD} in parallel... "
+echo -n "Launching test suites (${SUITES[*]}) in parallel... "
 _start_time=$(date +%s)
 for suite in "${SUITES[@]}"; do
+   suite_sn=$(suite_sn "$suite")
+   if ! suite_sn_only "$suite"; then
+       ./typhoon \
+           --cpus 1 \
+           --hash 256m \
+           --logfile "$LOGDIR/sd${SD}_${suite}.log" \
+           --batch \
+           --command "force; book name /nonexistent.book.bin; sd ${SD}; script $TESTS/${suite}.ep_" \
+           > "$LOGDIR/sd${SD}_${suite}.out" 2>&1 &
+   fi
    ./typhoon \
        --cpus 1 \
        --hash 256m \
-       --logfile "$LOGDIR/sd${SD}_${suite}.log" \
-       --batch \
-       --command "force; book name /nonexistent.book.bin; sd ${SD}; script $TESTS/${suite}.ep_" \
-       > "$LOGDIR/sd${SD}_${suite}.out" 2>&1 &
-   ./typhoon \
-       --cpus 1 \
-       --hash 256m \
-       --logfile "$LOGDIR/sn${SN}_${suite}.log" \
+       --logfile "$LOGDIR/sn${suite_sn}_${suite}.log" \
        --batch \
-       --command "force; book name /nonexistent.book.bin; sn ${SN}; script $TESTS/${suite}.ep_" \
-       > "$LOGDIR/sn${SN}_${suite}.out" 2>&1 &
+       --command "force; book name /nonexistent.book.bin; sn ${suite_sn}; script $TESTS/${suite}.ep_" \
+       > "$LOGDIR/sn${suite_sn}_${suite}.out" 2>&1 &
 done
 wait
 _elapsed=$(( $(date +%s) - _start_time ))
@@ -86,15 +119,19 @@ print_stats_table() {
 }
 
 for suite in "${SUITES[@]}"; do
-    echo "----[ $suite sd=${SD} ]------------------------------------------------------------------"
-    print_stats_table \
-        <(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sd${SD}_${suite}.out") \
-        <(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sd${SD}_${suite}.out")
-    echo
-    echo "----[ $suite sn=${SN} ]------------------------------------------------------------------"
+    suite_sn=$(suite_sn "$suite")
+    suite_sn_label=$(suite_sn_label "$suite")
+    if ! suite_sn_only "$suite"; then
+        echo "----[ $suite sd=${SD} ]------------------------------------------------------------------"
+        print_stats_table \
+            <(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sd${SD}_${suite}.out") \
+            <(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sd${SD}_${suite}.out")
+        echo
+    fi
+    echo "----[ $suite sn=${suite_sn} ]------------------------------------------------------------------"
     print_stats_table \
-        <(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sn${SN_LABEL}_${suite}.out") \
-        <(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sn${SN}_${suite}.out")
+        <(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sn${suite_sn_label}_${suite}.out") \
+        <(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sn${suite_sn}_${suite}.out")
     echo
 done
 echo