Add ecm_late.ep_ (a 25-position slow-to-confirm ECM subset) and wire run_tests.sh to run it sn-only, as a search/extensions-quality metric distinct from the curated suites' solve-count gate.

Positions are ECM problems where the correct move only becomes -- and stays -- the reported best move well into the search (5+ seconds, confirmed stable for 2+ further iterations, non-zero final score) across a majority of 8 independent full-suite runs; derived from mining 8 full 879-position ECM runs at 20s/move for exactly this pattern, then filtering out the "engine got lucky on the last iteration" and "drawn by repetition" false positives that a naive late-solve detector picks up (cross-checked against tests/ECM.md's existing flaky-position list).

sn-only (no sd sweep) because a fixed depth on 1 thread, with no cutoff-sharing across threads to shrink the tree, can blow up unpredictably on the sharpest of these positions; a fixed 15M-node budget keeps wall-clock bounded (~5min/suite) while staying in the right ballpark of the per-thread node share implied by the 8-thread discovery data.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

2 files changed, 157 insertions, 20 deletions

diff --git a/src/run_tests.sh b/src/run_tests.sh
index 28e8ffe..8f2d3f1 100755
--- a/src/run_tests.sh
+++ b/src/run_tests.sh
@@ -17,31 +17,64 @@ fi
 # sd (fixed depth) is the right choice when judging whether a pruning/
 # ordering change makes the tree smaller or larger for the same search
 # effort; sn (fixed node budget) is the right choice when judging how many
-# positions solve within a fixed cost. Override via SD/SN env vars.
+# positions solve within a fixed cost. Override via SD/SN env vars --
+# this applies to every suite passed on the command line.
+#
+# ecm_late.ep_ (see its header comment) is a different shape of suite --
+# curated for positions where the *correct* move is only confirmed well
+# into the search, specifically so avg-solution-time/total-nodecount from
+# the stats block below can be read as an extensions/search-quality
+# metric ("does the engine confirm these tactics sooner, in aggregate")
+# rather than a solve-count gate. It's sn-only (no sd sweep): the
+# discovery data behind it was measured on an 8-thread box (avg ~13 ply /
+# ~70M total nodes across all 8 threads) and a fixed *depth* on 1 thread,
+# with no cutoff-sharing across threads to shrink the tree, can blow up
+# unpredictably on the sharpest of these positions -- a fixed *node*
+# budget bounds wall-clock regardless. 15M nodes (~12s/position at this
+# box's ~1.3Mnps single-thread rate, ~5 min for all 25 positions) is
+# picked to stay in "quick ad hoc iteration" territory while still being
+# in the right ballpark of the per-thread node share implied by the
+# 8-thread discovery data -- not a rigorously derived number, adjust via
+# SN if it's cutting off too many positions before they confirm.
 SD="${SD:-10}"
 SN="${SN:-5000000}"
 SN_LABEL="${SN_LABEL:-5m}"
+ECM_LATE_SN=15000000
+ECM_LATE_SN_LABEL=15m
+
+suite_sn_only() {
+    [ "$1" = "ecm_late" ]
+}
+suite_sn() {
+    if [ "$1" = "ecm_late" ]; then echo "$ECM_LATE_SN"; else echo "$SN"; fi
+}
+suite_sn_label() {
+    if [ "$1" = "ecm_late" ]; then echo "$ECM_LATE_SN_LABEL"; else echo "$SN_LABEL"; fi
+}
 
 /bin/rm -rf "${LOGDIR}"
 mkdir -p "${LOGDIR}"
 
-echo -n "Launching test suites (${SUITES[*]}) @ sn=${SN} and sd=${SD} in parallel... "
+echo -n "Launching test suites (${SUITES[*]}) in parallel... "
 _start_time=$(date +%s)
 for suite in "${SUITES[@]}"; do
+   suite_sn=$(suite_sn "$suite")
+   if ! suite_sn_only "$suite"; then
+       ./typhoon \
+           --cpus 1 \
+           --hash 256m \
+           --logfile "$LOGDIR/sd${SD}_${suite}.log" \
+           --batch \
+           --command "force; book name /nonexistent.book.bin; sd ${SD}; script $TESTS/${suite}.ep_" \
+           > "$LOGDIR/sd${SD}_${suite}.out" 2>&1 &
+   fi
    ./typhoon \
        --cpus 1 \
        --hash 256m \
-       --logfile "$LOGDIR/sd${SD}_${suite}.log" \
-       --batch \
-       --command "force; book name /nonexistent.book.bin; sd ${SD}; script $TESTS/${suite}.ep_" \
-       > "$LOGDIR/sd${SD}_${suite}.out" 2>&1 &
-   ./typhoon \
-       --cpus 1 \
-       --hash 256m \
-       --logfile "$LOGDIR/sn${SN}_${suite}.log" \
+       --logfile "$LOGDIR/sn${suite_sn}_${suite}.log" \
        --batch \
-       --command "force; book name /nonexistent.book.bin; sn ${SN}; script $TESTS/${suite}.ep_" \
-       > "$LOGDIR/sn${SN}_${suite}.out" 2>&1 &
+       --command "force; book name /nonexistent.book.bin; sn ${suite_sn}; script $TESTS/${suite}.ep_" \
+       > "$LOGDIR/sn${suite_sn}_${suite}.out" 2>&1 &
 done
 wait
 _elapsed=$(( $(date +%s) - _start_time ))
@@ -86,15 +119,19 @@ print_stats_table() {
 }
 
 for suite in "${SUITES[@]}"; do
-    echo "----[ $suite sd=${SD} ]------------------------------------------------------------------"
-    print_stats_table \
-        <(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sd${SD}_${suite}.out") \
-        <(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sd${SD}_${suite}.out")
-    echo
-    echo "----[ $suite sn=${SN} ]------------------------------------------------------------------"
+    suite_sn=$(suite_sn "$suite")
+    suite_sn_label=$(suite_sn_label "$suite")
+    if ! suite_sn_only "$suite"; then
+        echo "----[ $suite sd=${SD} ]------------------------------------------------------------------"
+        print_stats_table \
+            <(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sd${SD}_${suite}.out") \
+            <(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sd${SD}_${suite}.out")
+        echo
+    fi
+    echo "----[ $suite sn=${suite_sn} ]------------------------------------------------------------------"
     print_stats_table \
-        <(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sn${SN_LABEL}_${suite}.out") \
-        <(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sn${SN}_${suite}.out")
+        <(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sn${suite_sn_label}_${suite}.out") \
+        <(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sn${suite_sn}_${suite}.out")
     echo
 done
 echo
diff --git a/tests/ecm_late.ep_ b/tests/ecm_late.ep_
new file mode 100644
index 0000000..ef8facd
--- /dev/null
+++ b/tests/ecm_late.ep_
@@ -0,0 +1,100 @@
+setboard 2rr2k1/1b3p1p/p4qpb/2R1n3/3p4/BP2P3/P3QPPP/3R1BKN b - -
+solution Rxc5
+id "ECM.016"
+go
+setboard 8/p3q1kp/1p2Pnp1/3pQ3/2pP4/1nP3N1/1B4PP/6K1 w - -
+solution Ba3
+id "ECM.146"
+go
+setboard 2r2rk1/1p1q1ppp/1p3n2/3p1N2/4n3/1N3Q2/PPP2PPP/R2R2K1 w - -
+solution Rxd5
+id "ECM.196"
+go
+setboard 2r1qrk1/pp1b1ppp/4pn2/n1b5/8/2NQ1NP1/PP1BPPBP/R2R2K1 w - -
+solution b4
+id "ECM.297"
+go
+setboard r2qrbk1/1p4pb/2n2p1p/pNpn4/2N5/PP1PPQPB/1B5P/2R2RK1 w - -
+solution Nbd6
+id "ECM.313"
+go
+setboard rkb2qr1/1p5p/pQ1bp3/3p2B1/5n2/3B1N2/PP3PPP/2R2RK1 w - -
+solution Rxc8+
+id "ECM.332"
+go
+setboard 1r2r1k1/1pqb2pp/p2bp1n1/2p3BQ/3pN3/1P1P2P1/P1P3BP/4RRK1 w - -
+solution Bd8
+id "ECM.370"
+go
+setboard r3bb2/P1q3k1/Q2p3p/2pPp1pP/2B1P3/2B5/6P1/R5K1 w - -
+solution Bxe5+
+id "ECM.379"
+go
+setboard 3r1r2/1b4bk/p1n3pp/1p2p3/4P3/q1P1QNB1/B4RPP/5RK1 w - -
+solution Bh4
+id "ECM.438"
+go
+setboard 2k3r1/1ppr3p/p1pB2q1/6n1/N3Pp2/8/PPP1Q1PP/3R1RK1 b - -
+solution f3
+id "ECM.448"
+go
+setboard r1bqr2k/ppp2p1p/5b2/3Pn1p1/1PP1Bp2/8/PB2N1PP/R2Q1RK1 w - -
+solution Nxf4
+id "ECM.504"
+go
+setboard r3k2r/1p1b1p2/p3p3/2b3p1/4N1n1/1B5P/PPP3P1/R1B2R1K b kq -
+solution Bc6
+id "ECM.506"
+go
+setboard 6k1/1q2rpp1/p6p/P7/1PB1n3/5Q2/6PP/5R1K w - -
+solution b5
+id "ECM.615"
+go
+setboard r5r1/8/bRP1pk1p/3pNp2/5P2/7P/PR4P1/6K1 w - -
+solution Rxa6
+id "ECM.616"
+go
+setboard 2r1rbk1/p1Bq1ppp/Ppn1b3/1Npp4/B7/3P2Q1/1PP2PPP/R4RK1 w - -
+solution Nxa7
+id "ECM.635"
+go
+setboard r2qrbk1/5ppp/pn1p4/np2P1P1/3p4/5N2/PPB2PP1/R1BQR1K1 w - -
+solution Bxh7+
+id "ECM.654"
+go
+setboard r2q1rk1/3n1ppp/8/1pbP2P1/p1N4P/PnBBPQ2/5P2/R3K2R w KQ -
+solution Bxh7+
+id "ECM.697"
+go
+setboard r3r1k1/pp3pp1/3p4/2q4p/2P5/1PB2Q1P/n4PP1/3R1RK1 w - -
+solution Bxg7
+id "ECM.713"
+go
+setboard r2rn3/1p3pk1/p1pNn1pp/q3P3/P7/1PN4P/2QR1PP1/3R2K1 w - -
+solution Nxf7
+id "ECM.764"
+go
+setboard rn2r1k1/pp3p1p/3P2p1/2p1bbB1/2B5/2N5/Pq4PP/R2Q1RK1 w - -
+solution Bxf7+
+id "ECM.767"
+go
+setboard 2r1r1k1/pb1n1pp1/1p1qpn1p/4N1B1/2PP4/3B4/P2Q1PPP/3RR1K1 w - -
+solution Bxh6
+id "ECM.789"
+go
+setboard 4rrk1/p5pp/1p4q1/3p2n1/2pP2P1/P1P1PP2/3B2K1/1R2QR2 b - -
+solution Nxf3
+id "ECM.824"
+go
+setboard r1bq1rk1/1p2bppp/p3p3/n3P3/4N3/1P1P1N2/PB4PP/R3QR1K w - -
+solution Nf6+
+id "ECM.829"
+go
+setboard r1bq1rk1/ppp3p1/7p/3P2n1/2PQ1p2/1N5P/PPP2PPK/R1B2R2 b - -
+solution Nf3+
+id "ECM.831"
+go
+setboard 6k1/1R4pp/2p5/8/P1rNp3/6Pb/4PK2/8 w - -
+solution a5
+id "ECM.878"
+go