summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorScott Gasch <[email protected]>2026-10-20 09:26:44 -0700
committerScott Gasch <[email protected]>2026-10-20 09:26:44 -0700
commitc8c6bbc2c1a103e5b07b079a0f8bdf66c7724d80 (patch)
treeb6c2137c9ac577fa83b1e948fa0311d53dfe445d /src
parent3b5d3677bb14ae14728a1c4b221176bb83667d92 (diff)
Wire NNUE eval into search, build the training pipeline, and add a fast
move/eval-agreement benchmark against Stockfish (tests/lop) Implements the architecture decisions in src/docs/NNUE.md: HalfKAv2_hm feature set, int16/int8 quantization, per-ply accumulator backup for king moves, and a lazy-eval-ready integration into search.c/eval.c. Adds the full annotate_pgns/ training pipeline (extraction, K-fit/ labeling, quiet filtering, tensor building, training, export) plus an end-to-end orchestrator (run_nnue_pipeline.py) and validation tooling (sanity_check_net.py, compare_vs_stockfish_net.py). Adds tests/lop: a 10k-position list-of-positions benchmark comparing engine eval rank and move choice against pre-computed Stockfish/lc0 references, much cheaper than a full self-play match for day-to-day progress checks. First sd10-matched self-play result against classic HCE: LLR +2.28 (H1=+2.94), -72=119+169 over 360 games, confirming the net is "smarter per node" at matched depth. A real-time (--st 4) match is currently weaker (candidate loses ~3 plies of depth to a ~3.3-3.7x nps gap against an unrecalibrated search), consistent with NNUE.md's adoption-criterion section -- pruning/reduction constants still need retuning for the new eval's score scale before a fair real-time comparison is possible. Co-Authored-By: Claude Sonnet 5 <[email protected]>
Diffstat (limited to 'src')
-rw-r--r--src/GNUmakefile14
-rw-r--r--src/TODO6
-rwxr-xr-xsrc/chess.h163
-rw-r--r--src/command.c196
-rw-r--r--src/docs/NNUE.md1605
-rw-r--r--src/eval.c347
-rw-r--r--src/eval_tune/build_cross_sectional_all.py228
-rw-r--r--src/eval_tune/compare_vs_crafty.py597
-rwxr-xr-xsrc/eval_tune/match_play.py275
-rwxr-xr-xsrc/eval_tune/nnue_vs_head.sh219
-rw-r--r--src/eval_tune/relabel_selfplay_pgn.py112
-rwxr-xr-xsrc/eval_tune/stockfish_training_corpus.sh13
-rwxr-xr-xsrc/eval_tune/test_vs_head.sh94
-rwxr-xr-xsrc/eval_tune/typhoon_hce_training_corpus.sh173
-rwxr-xr-xsrc/eval_tune/typhoon_nnue_training_corpus.sh224
-rw-r--r--src/main.c48
-rw-r--r--src/nnueaccum.c30
-rw-r--r--src/nnueeval.c406
-rw-r--r--src/search.c162
-rwxr-xr-xsrc/split_ep.py69
-rw-r--r--src/testnnue.c31
-rw-r--r--src/util.c5
-rw-r--r--src/vars.c10
23 files changed, 4818 insertions, 209 deletions
diff --git a/src/GNUmakefile b/src/GNUmakefile
index bbe79d1..7dbb143 100644
--- a/src/GNUmakefile
+++ b/src/GNUmakefile
@@ -98,6 +98,20 @@ ifdef GPROF
PROFILE += -pg
endif
+# Scoped ISA opt-in for NNUE SIMD work (int8/int16 dot products, and
+# the FC head's existing float32 loops which clang already
+# autovectorizes to SSE only without this). Default ON since 2026-10-16
+# -- confirmed a real, measured win for both classic HCE (~3%) and
+# (once real NNUE weights are loaded) the NNUE FC head (~7%, see
+# docs/NNUE.md), and the build box (Zen 2) has full-width 256-bit
+# AVX2 + FMA3. Pass AVX2=0 explicitly to disable (e.g. building on a
+# pre-Haswell/pre-Zen box, 2013 and earlier) -- unlike -march=native,
+# this doesn't assume the BUILD box matches the RUN box, but it does
+# assume AVX2 availability by default now rather than opt-in.
+ifneq ($(AVX2),0)
+ PROFILE += -mavx2 -mfma
+endif
+
ifdef DEBUG
PROFILE += -g -DDEBUG
BINARY = _typhoon
diff --git a/src/TODO b/src/TODO
index 926f67b..0d2251b 100644
--- a/src/TODO
+++ b/src/TODO
@@ -71,3 +71,9 @@ Valgrind in debug_smoke_test.sh.
Remove all line-ending whitespace in one commit.
+Fix the input queue --batch mode one thing after the script; like this breaks it.
+
+Add nullmoves skipped to the output. Redo the output in general.
+
+Remove "pure Newton's square root method" stuff and get math.h loaded.
+
diff --git a/src/chess.h b/src/chess.h
index cfe1d4e..3e65e7f 100755
--- a/src/chess.h
+++ b/src/chess.h
@@ -1630,22 +1630,25 @@ MOVE_SCORING_SCHEME;
// ever applied where the accumulator is actually read (the FC head),
// never where it's maintained.
//
-// Deliberately float, not yet the int16 the final architecture
-// decision calls for (NNUE.md's "Quantization" section) -- that needs
-// a calibrated per-layer scale from the quantization-aware export
-// pass, which doesn't exist yet. This branch's job is proving the
-// incremental-maintenance plumbing (king-move backup/restore nesting,
-// capture/promotion edge cases) is correct; swapping the element type
-// is a later, decoupled, mechanical follow-up once real quantized
-// weights exist -- not something to conflate with debugging the
-// harder plumbing now.
+// Int32, not float: the embedding table (g_rgfNnueWeight) is
+// quantized to SHORT at FT_SCALE (see export_nnue_weights.py /
+// g_fNnueFtScale), and summing up to 32 active features' SHORT rows
+// can exceed SHORT's range even though each individual row value
+// fits -- measured worst-case real accumulator magnitude (NNUE.md's
+// "Quantization safety-checked" note) is ~26 in float-equivalent
+// units, i.e. ~106,500 at the chosen scale, well past MAX_SHORT.
+// INT headroom makes this a non-issue. Converted back to float
+// (divided by g_fNnueFtScale) only where the accumulator is actually
+// read (the FC head, nnueeval.c's NnueForward) -- never where it's
+// maintained, so incremental MakeMove/UnmakeMove updates stay purely
+// additive integer arithmetic (exact, no float drift).
#define NNUE_ACCUMULATOR_SIZE (256) // L1, nnue_model.py
#define NNUE_L2 (32) // nnue_model.py's L2
#define NNUE_L3 (32) // nnue_model.py's L3
typedef struct _NNUE_ACCUMULATOR
{
- float rgfValue[NNUE_ACCUMULATOR_SIZE];
+ INT rgiValue[NNUE_ACCUMULATOR_SIZE];
}
NNUE_ACCUMULATOR;
@@ -2127,6 +2130,18 @@ typedef struct _GAME_OPTIONS
FLAG fStatusLine;
FLAG fFastScript;
INT iResignThreshold;
+ // Minimum elapsed time (seconds, since this move's search started)
+ // before UtilPrintPV (util.c) will emit a PV line at all -- default
+ // 0.35 matches the historical hardcoded constant. 2026-10-16: a
+ // search that finishes a move's ENTIRE iterative-deepening loop
+ // faster than this (common at low sd/st, e.g. scripted self-play
+ // corpus generation) gets NO PV line at all, not a delayed one --
+ // match_play.py's "no eval captured for this move" case. Exposed as
+ // a runtime "set PostThrottle <seconds>" variable (vars.c) rather
+ // than hand-editing the constant, specifically so a fast scripted
+ // run can set it to 0 without changing default interactive/xboard
+ // behavior or risking a forgotten permanent edit.
+ double dPostThrottleSec;
enum
{
@@ -2307,13 +2322,20 @@ NnueFeatureIndex(ULONG uPerspective, COOR cKingOwn, COOR cPieceSquare, PIECE pPi
return idx;
}
-// Placeholder embedding table -- NOT yet loaded from a real trained
-// checkpoint (see docs/NNUE.md's "not yet built: a weight-loading
-// mechanism"). Zero-initialized by C's own static/global default, so
-// every accumulator value is 0 until real weights are loaded; this is
-// enough to prove the incremental-maintenance plumbing is correct.
-extern float g_rgfNnueWeight[NNUE_DIMENSIONS][NNUE_ACCUMULATOR_SIZE];
-extern float g_rgfNnueBias[NNUE_ACCUMULATOR_SIZE];
+// Embedding table, quantized to SHORT (see docs/NNUE.md's
+// "Quantization plan" -- this is the per-move hot path table that
+// dominates memory bandwidth; halving it from float to SHORT is the
+// whole point). Zero-initialized by C's own static/global default
+// until a real exported checkpoint is loaded via NnueLoadWeights.
+// g_fNnueFtScale is the fixed-point scale applied by
+// export_nnue_weights.py's QuantizeFt before rounding to SHORT;
+// NnueForward divides it back out when converting the accumulator
+// (NNUE_ACCUMULATOR.rgiValue, INT) to float for the FC head. Defaults
+// to 1.0f so the all-zero placeholder table behaves sanely (0/1.0 ==
+// 0) before any real weights are loaded.
+extern SHORT g_rgfNnueWeight[NNUE_DIMENSIONS][NNUE_ACCUMULATOR_SIZE];
+extern SHORT g_rgfNnueBias[NNUE_ACCUMULATOR_SIZE];
+extern float g_fNnueFtScale;
void
NnueAccumAddFeature(NNUE_ACCUMULATOR *pAccum, ULONG uPerspective,
@@ -2336,12 +2358,26 @@ NnueAccumRecompute(NNUE_ACCUMULATOR *pAccum, POSITION *pos, ULONG uPerspective);
// (callers should, before trusting its output for anything beyond
// plumbing validation).
//
-extern float g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE];
+// FC head weights, quantized to CHAR (int8, signed) per
+// docs/NNUE.md's "Quantization plan" step 2 -- biases stay float32
+// (tiny arrays, added post-dequant, not worth quantizing).
+// g_rgfNnueActScale[0..2] (AS1,AS2,AS3) are the per-stage uint8
+// fixed-point scales used to requantize each ReLU'd activation before
+// the next layer's int8 dot product; g_rgfNnueWeightScale[0..2]
+// (WS1,WS2,WS3) are the matching per-layer weight scales. Both are
+// read from the export file's header (NnueLoadWeights) -- see
+// annotate_pgns/export_nnue_weights.py's CalibrateActivationScales,
+// which derives AS1-3 from real sampled positions (not a theoretical
+// bound) since these activations' true range isn't knowable from the
+// weights alone.
+extern CHAR g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE];
extern float g_rgfNnueFc1Bias[NNUE_L2];
-extern float g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2];
+extern CHAR g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2];
extern float g_rgfNnueFc2Bias[NNUE_L3];
-extern float g_rgfNnueFc3Weight[NNUE_L3];
+extern CHAR g_rgfNnueFc3Weight[NNUE_L3];
extern float g_rgfNnueFc3Bias;
+extern float g_rgfNnueActScale[3];
+extern float g_rgfNnueWeightScale[3];
extern FLAG g_fNnueWeightsLoaded;
extern FLAG g_fUseNnueEval;
@@ -2351,6 +2387,41 @@ NnueLoadWeights(CHAR *szFilename);
SCORE
NnueForward(SEARCHER_THREAD_CONTEXT *ctx);
+void
+NnueDumpActivationStats(SEARCHER_THREAD_CONTEXT *ctx);
+
+// Calibration-only lazy-eval probe (docs/NNUE.md's "Lazy-evaluation
+// tiering plan") -- a linear regression fit over fc1's own input (the
+// ReLU'd, concatenated accumulator), separate from the production
+// .nnue weight file/format entirely. NnueProbeScore is cheap (one
+// 512-wide dot product, no fc1/fc2/fc3) and exists purely to measure
+// how well it tracks NnueForward's real output, via the
+// CALIBRATE_MARGIN_SAFETY harness in eval.c -- not yet used to skip
+// any real computation.
+extern FLAG g_fNnueProbeLoaded;
+
+// Separate from g_fUseNnueEval/g_fNnueProbeLoaded -- controls whether
+// Eval()'s NNUE short-circuit actually TRUSTS the probe-based gate to
+// skip NnueForward (vs. just measuring it under CALIBRATE_MARGIN_
+// SAFETY). Defaults FALSE, same convention as g_fUseNnueEval itself --
+// an explicit opt-in, not auto-enabled just because weights/probe are
+// loaded. See docs/NNUE.md's "Lazy-evaluation tiering plan".
+extern FLAG g_fUseNnueLazyProbe;
+
+// Runtime experimentation knobs -- see nnueeval.c's comment. Flat
+// override for the lazy probe's margin (< 0 = use the calibrated
+// per-bucket table) and the super-lazy (material-only, no net)
+// pre-filter's own on/off + margin.
+extern INT g_iNnueLazyMarginOverride;
+extern FLAG g_fUseNnueSuperLazy;
+extern SCORE g_iNnueSuperLazyMargin;
+
+FLAG
+NnueProbeLoadWeights(CHAR *szFilename);
+
+SCORE
+NnueProbeScore(SEARCHER_THREAD_CONTEXT *ctx);
+
#ifdef PERF_COUNTERS
#define INC(x) ((x) += 1)
#else
@@ -3925,7 +3996,19 @@ IsDraw(SEARCHER_THREAD_CONTEXT *ctx);
#define EVAL_POSITIONAL_SOURCE_FULL (0)
#define EVAL_POSITIONAL_SOURCE_LAZY (1)
#define EVAL_POSITIONAL_SOURCE_SUPERLAZY (2)
-#define EVAL_POSITIONAL_SOURCE_COUNT (3)
+// Added 2026-10-16 (docs/NNUE.md's "Lazy-evaluation tiering plan"):
+// Eval()'s NNUE short-circuit never set uLastPositionalSource at all
+// before this -- every NNUE-sourced score silently inherited whatever
+// stale value a prior classic-Eval() call left in that ply's PLY_INFO
+// slot (or the C zero-init default, EVAL_POSITIONAL_SOURCE_FULL,
+// if none ever ran), so QSearch futility pruning was using an
+// unexamined, accidentally-conservative margin for every NNUE score,
+// with no distinction between the full forward pass (exact) and the
+// lazy accumulator-probe exit (real, measured error). These two give
+// each NNUE source its own tier, same as the three classic ones.
+#define EVAL_POSITIONAL_SOURCE_NNUE_FULL (3)
+#define EVAL_POSITIONAL_SOURCE_NNUE_LAZY (4)
+#define EVAL_POSITIONAL_SOURCE_COUNT (5)
// Per-tier qsearch futility base margins -- see search.c's
// FUTILITY_BASE_MARGIN_BY_SOURCE for the full derivation. Named here
@@ -3967,6 +4050,44 @@ IsDraw(SEARCHER_THREAD_CONTEXT *ctx);
// qsearchfutility` if iPositional's construction changes again.
#define FUTILITY_BASE_MARGIN_LAZY (350)
#define FUTILITY_BASE_MARGIN_SUPERLAZY (175)
+// Placeholder starting points, 2026-10-16 -- NOT yet calibrated via
+// `calibrate qsearchfutility` against real data (see
+// EVAL_POSITIONAL_SOURCE_NNUE_FULL/NNUE_LAZY's own comment). NNUE_FULL
+// starts equal to classic FULL (both are exact, non-approximate static
+// evals -- a reasonable prior pending real measurement); NNUE_LAZY
+// starts equal to classic LAZY (both are an approximate exit one tier
+// down from exact). Re-derive both for real before trusting them.
+// 2026-10-16: first real data (twic_sample.ep_, 1500 positions, sd 6)
+// showed NNUE_FULL's 7.41% surprise rate at 450 in the checking-
+// capture/promo gate. Widening to 650 was then tried and tested as
+// NO-OP (identical 7.41%/n) -- root-caused to iFutility's MAX0 clamp
+// saturating this source's population 100.00% of the time (mean
+// pre-clamp -589.3) -- the margin constant is never actually read for
+// ~all of this test pool, so neither 450 nor 650 is "validated," both
+// are simply untested by this data. Reverted to 450 (matches classic
+// FULL) as the more defensible default pending real evidence either
+// way: when NNUE_FULL surprises DO occur (in whatever small
+// non-saturated slice exists, not yet isolated), their magnitude runs
+// smaller than classic FULL's own surprises (mean 477.5 vs 938.7cp,
+// >100cp 9.88% vs 28.74%, twic_sample.ep_ sd6) -- no evidence NNUE's
+// rare misses are more expensive, so no reason to prefer a wider,
+// unvalidated number over HCE's own long-trusted one.
+//
+// TODO: find and calibrate against the actual non-saturated NNUE_FULL
+// population specifically (likely nodes where iEval sits close to
+// iAlpha rather than comfortably above it -- the saturated case is
+// "iEval way above iAlpha", so look at the complement) before trusting
+// any NNUE_FULL margin as real, NNUE-specific-validated rather than a
+// borrowed HCE default.
+//
+// NNUE_LAZY=200 (tightened from 350) IS genuinely validated -- real
+// headroom confirmed (only 24.76% of its population even hits the
+// clamp, unlike NNUE_FULL's 100%), surprise rate landed in a sane
+// 1.3-2.1% range on real data, and its own surprise magnitudes
+// (mean 5085.3, >100cp 19.45%) run well below classic regular lazy's
+// (mean 11030.2, >100cp 69.73%). Keep this one.
+#define FUTILITY_BASE_MARGIN_NNUE_FULL (450)
+#define FUTILITY_BASE_MARGIN_NNUE_LAZY (200)
// Measured: disabling this entirely (see lmr_testing/RESULTS.md) is a
// clear net loss across ringers/confident_quick/hard_quick, so IID itself
diff --git a/src/command.c b/src/command.c
index 6e11d90..1767e98 100644
--- a/src/command.c
+++ b/src/command.c
@@ -819,9 +819,13 @@ Return value:
if (argc < 2)
{
Trace("Usage: nnue load <filename>\n"
+ " nnue probeload <filename>\n"
" nnue eval\n"
+ " nnue stats\n"
" nnue on\n"
- " nnue off\n");
+ " nnue off\n"
+ " nnue lazyprobeon\n"
+ " nnue lazyprobeoff\n");
return;
}
@@ -843,6 +847,28 @@ Return value:
return;
}
+ if (!STRCMPI(argv[1], "probeload"))
+ {
+ // Loads the experimental lazy-eval probe's weights (docs/
+ // NNUE.md's "Lazy-evaluation tiering plan") -- a separate,
+ // calibration-only weight file, not the production .nnue
+ // format NnueLoadWeights reads.
+ if (argc < 3)
+ {
+ Trace("Error (missing argument)\n");
+ return;
+ }
+ if (!NnueProbeLoadWeights(argv[2]))
+ {
+ Trace("Error loading NNUE probe weights file.\n");
+ }
+ else
+ {
+ Trace("Loaded NNUE probe weights from \"%s\"\n", argv[2]);
+ }
+ return;
+ }
+
if (!STRCMPI(argv[1], "eval"))
{
if (!g_fNnueWeightsLoaded)
@@ -880,6 +906,34 @@ Return value:
return;
}
+ if (!STRCMPI(argv[1], "stats"))
+ {
+ // Diagnostic: per-layer activation saturation report. See
+ // NnueDumpActivationStats's routine description (nnueeval.c).
+ if (!g_fNnueWeightsLoaded)
+ {
+ Trace("Error: no NNUE weights loaded -- "
+ "try 'nnue load <file>' first.\n");
+ return;
+ }
+ if (NULL == ctx)
+ {
+ ctx = SystemAllocateMemory(sizeof(SEARCHER_THREAD_CONTEXT));
+ if (NULL == ctx)
+ {
+ Trace("Out of memory.\n");
+ return;
+ }
+ InitializeSearcherContext(pos, ctx);
+ }
+ else
+ {
+ ReInitializeSearcherContext(pos, ctx);
+ }
+ NnueDumpActivationStats(ctx);
+ return;
+ }
+
if (!STRCMPI(argv[1], "on"))
{
if (!g_fNnueWeightsLoaded)
@@ -900,10 +954,148 @@ Return value:
return;
}
+ if (!STRCMPI(argv[1], "lazyprobeon"))
+ {
+ if (!g_fNnueProbeLoaded)
+ {
+ Trace("Error: no NNUE probe weights loaded -- "
+ "try 'nnue probeload <file>' first.\n");
+ return;
+ }
+ g_fUseNnueLazyProbe = TRUE;
+ Trace("NNUE lazy probe gate enabled.\n");
+ return;
+ }
+
+ if (!STRCMPI(argv[1], "lazyprobeoff"))
+ {
+ g_fUseNnueLazyProbe = FALSE;
+ Trace("NNUE lazy probe gate disabled.\n");
+ return;
+ }
+
+ if (!STRCMPI(argv[1], "lazymargin"))
+ {
+ // Flat override for every material bucket's lazy-probe margin
+ // -- experimentation knob, see nnueeval.c. Pass a negative
+ // number (or omit) to go back to the calibrated table.
+ if (argc < 3)
+ {
+ g_iNnueLazyMarginOverride = -1;
+ Trace("NNUE lazy margin override cleared (back to calibrated table).\n");
+ return;
+ }
+ g_iNnueLazyMarginOverride = atoi(argv[2]);
+ Trace("NNUE lazy margin override set to %d cp.\n", g_iNnueLazyMarginOverride);
+ return;
+ }
+
+ if (!STRCMPI(argv[1], "superlazyon"))
+ {
+ if (argc < 3)
+ {
+ Trace("Usage: nnue superlazyon <margin cp>\n");
+ return;
+ }
+ g_iNnueSuperLazyMargin = (SCORE)atoi(argv[2]);
+ g_fUseNnueSuperLazy = TRUE;
+ Trace("NNUE super-lazy (material-only) pre-filter enabled, margin=%d cp.\n",
+ g_iNnueSuperLazyMargin);
+ return;
+ }
+
+ if (!STRCMPI(argv[1], "superlazyoff"))
+ {
+ g_fUseNnueSuperLazy = FALSE;
+ Trace("NNUE super-lazy pre-filter disabled.\n");
+ return;
+ }
+
+ if (!STRCMPI(argv[1], "bench"))
+ {
+ // Scratch diagnostic, not a shipped feature -- times
+ // NnueAccumAddFeature/RemoveFeature (the per-MakeMove hot
+ // path) against NnueForward/NnueProbeScore (the per-eval hot
+ // path) so we know which one actually dominates before
+ // spending effort hand-vectorizing either. See docs/NNUE.md.
+ const ULONG N = 2000000;
+ ULONG i;
+ UINT64 u64Start, u64AddRemove, u64Forward, u64Probe;
+ NNUE_ACCUMULATOR acc;
+ COOR cKing = pos->cKingSq[WHITE];
+ COOR cSq = FILE_RANK_TO_COOR(3, 4);
+ PIECE p = pos->rgSquare[FILE_RANK_TO_COOR(0, 1)].pPiece;
+
+ memset(&acc, 0, sizeof(acc));
+ u64Start = SystemReadTimeStampCounter();
+ for (i = 0; i < N; i++)
+ {
+ NnueAccumAddFeature(&acc, WHITE, cKing, cSq, p);
+ NnueAccumRemoveFeature(&acc, WHITE, cKing, cSq, p);
+ }
+ u64AddRemove = SystemReadTimeStampCounter() - u64Start;
+
+ if (!g_fNnueWeightsLoaded)
+ {
+ Trace("nnue bench: load weights first ('nnue load <file>') "
+ "to also time the forward pass.\n");
+ Trace("Add+Remove (x%lu): %" COMPILER_LONGLONG_UNSIGNED_FORMAT
+ " cycles total, %.1f cycles/call\n",
+ N, u64AddRemove, (double)u64AddRemove / (2.0 * N));
+ return;
+ }
+
+ if (NULL == ctx)
+ {
+ ctx = SystemAllocateMemory(sizeof(SEARCHER_THREAD_CONTEXT));
+ InitializeSearcherContext(pos, ctx);
+ }
+ else
+ {
+ ReInitializeSearcherContext(pos, ctx);
+ }
+
+ u64Start = SystemReadTimeStampCounter();
+ for (i = 0; i < N; i++)
+ {
+ volatile SCORE s = NnueForward(ctx);
+ (void)s;
+ }
+ u64Forward = SystemReadTimeStampCounter() - u64Start;
+
+ u64Probe = 0;
+ if (g_fNnueProbeLoaded)
+ {
+ u64Start = SystemReadTimeStampCounter();
+ for (i = 0; i < N; i++)
+ {
+ volatile SCORE s = NnueProbeScore(ctx);
+ (void)s;
+ }
+ u64Probe = SystemReadTimeStampCounter() - u64Start;
+ }
+
+ Trace("nnue bench (N=%lu):\n", N);
+ Trace(" Add+Remove pair : %10.1f cycles/call\n",
+ (double)u64AddRemove / (2.0 * N));
+ Trace(" NnueForward : %10.1f cycles/call\n",
+ (double)u64Forward / (double)N);
+ if (g_fNnueProbeLoaded)
+ {
+ Trace(" NnueProbeScore : %10.1f cycles/call\n",
+ (double)u64Probe / (double)N);
+ }
+ return;
+ }
+
Trace("Usage: nnue load <filename>\n"
+ " nnue probeload <filename>\n"
" nnue eval\n"
" nnue on\n"
- " nnue off\n");
+ " nnue off\n"
+ " nnue lazyprobeon\n"
+ " nnue lazyprobeoff\n"
+ " nnue bench\n");
}
diff --git a/src/docs/NNUE.md b/src/docs/NNUE.md
index 0ab46ec..dbc5d54 100644
--- a/src/docs/NNUE.md
+++ b/src/docs/NNUE.md
@@ -1034,16 +1034,187 @@ interchangeable with real Stockfish `.nnue` binaries.
2,298,648 positions, 84,302 exact rows left unlabeled),
`games.holdout.labeled.jsonl` (3,005 games, 255,245 positions, 9,113
exact rows left unlabeled).
-- **Not yet built**: the Python training script itself (the loss
- function, quantization-aware export pass), the feature-indexing-spec
- cross-checker, any `SEARCHER_THREAD_CONTEXT`/accumulator plumbing
- (still correctly gated behind a dedicated branch, not started), and
- the lambda sweep/selection for blending `p_search`/`r_outcome`. The
+- **Update 2026-10-15: all of the below is now done** (see "Status
+ (2026-10-15)" further down for the full writeup) -- the training
+ script, feature-indexing cross-checker, accumulator plumbing, and
+ the first trained/exported/wired-in/match-played net all exist now.
+ **Still not done**: the lambda sweep/selection for blending
+ `p_search`/`r_outcome` (still correctly deferred to Phase B, per the
+ lambda decision above -- no new information has arrived that would
+ change that), and quantization (see "Quantization plan"). The
quiet-filter gap previously noted here (not yet applied to the
game-shaped/labeled files) is closed -- see `quiet_filter_games.py`
above -- and a future annotation run can skip the post-hoc step
entirely via `annotate.py --prefilter-quiet`.
+## Status (2026-10-15): training run finished, net wired into real search, first match-play result
+
+- **Phase A training run finished.** `annotate_pgns/nnue_train.py` against
+ the ~2.08M-row quiet-filtered pool above, 20 epochs, PyTorch on CPU
+ (per the Phase A local-CPU decision below) -- final train MSE 0.0065,
+ holdout MSE ~0.022. Checkpoints: `runs/sd10_twic_2M/nnue_ckpt/
+ ckpt_epoch000.pt` through `ckpt_epoch019.pt`.
+- **Feature-indexing cross-checker built and passing**
+ (`cross_check_features.py` against `nnue_feature_spec.py`'s Python
+ reference encoder and `src/nnuefeatures.c`'s C reference dump, 9 FENs,
+ all matching) -- closes milestone 1 below.
+- **Weight export pass built** (`export_nnue_weights.py`): flat binary
+ format, 4-byte `"NNU1"` magic, `<4I` dims header
+ (`NNUE_DIMENSIONS`/`L1`/`L2`/`L3`), then the eight weight/bias arrays
+ in a fixed order. `ckpt_epoch019.pt` exported to
+ `runs/sd10_twic_2M/nnue_ckpt/weights.nnue` (23MB).
+- **Accumulator plumbing built, tested, and proven fast** (on the
+ `nnue-accumulator-plumbing` branch, per milestone 3's branch
+ requirement) -- `SEARCHER_THREAD_CONTEXT` gained `sNnueAccum[2]`;
+ all 8 of `move.c`'s Lift/Place/Slide(WithoutSigs) primitives
+ maintain it incrementally (chose primitive-hooking over re-deriving
+ move-kind from `mv` after discussion -- these primitives already
+ couple tightly with Zobrist hashing, and `GenerateMoves`/
+ `MakeMove`/`UnmakeMove` are too hot to afford re-branching); king
+ moves trigger a full `NnueAccumRecompute` for their *own* perspective
+ only (every feature in that perspective is own-king-relative), with
+ a `PLY_INFO`-indexed backup/restore for `UnmakeMove`. `testnnue.c`'s
+ `TestNnueAccumulator` plays real games from 7 curated FENs chosen to
+ hit captures/promotions/en passant/castling/plain king moves within
+ 2 plies, comparing incremental maintenance against a from-scratch
+ `NnueAccumRecompute` after every move -- caught two real bugs on its
+ first-ever runs (a king-capture backup-ordering bug where the
+ backup was taken after the capturing `LiftPiece` instead of before
+ it; and, when the perf-gating fix below landed, the test's own
+ failure to set `g_fUseNnueEval` before exercising the gated code).
+ `setboard`'s downstream effect is covered by the same
+ `ReInitializeSearcherContext` choke point every `go`/`ponder` already
+ calls -- confirmed this is the one place that needs a hook, not
+ `SetRootPosition` itself.
+- **Load/call infrastructure and real search integration built**:
+ `NnueLoadWeights` (`nnueeval.c`) loads an exported `.nnue` file,
+ dimension-checked against the build's own constants; `NnueForward`
+ runs the real FC head (ReLU(accumulator) -> own-perspective-first
+ concat by side-to-move -> dense 2L1->L2->L3->1). `Eval()` now
+ short-circuits to `NnueForward` when `g_fUseNnueEval &&
+ g_fNnueWeightsLoaded`. Reachable via the `nnue load|eval|on|off`
+ command and a `--nnueweights <file>` CLI flag (the latter needed
+ because `match_play.py` spawns engines with fixed args -- no chance
+ to send interactive commands first).
+- **Critical performance regression found and fixed.** The accumulator
+ maintenance added to move.c's primitives ran unconditionally on the
+ (unverified, wrong) assumption that it was cheap -- cost ~50% of
+ engine throughput even with NNUE off (673k nps measured vs. the
+ expected 1.4-1.5M baseline). Gated all of it behind `g_fUseNnueEval`.
+ Confirmed via `bench`: classic HCE back to ~1.52M nps, NNUE eval a
+ fair ~410-411k nps.
+- **`test_vs_head.sh` extended with `NNUE_WEIGHTS`**: wraps the
+ candidate binary in a thin exec shim that always injects
+ `--nnueweights $NNUE_WEIGHTS`, since `match_play.py`'s `Engine` class
+ passes identical `engine_args` to both sides and spawns the given
+ path directly via `Popen` (no shell), so a shebang wrapper script in
+ the candidate's place works unmodified -- no `match_play.py` patch
+ needed.
+- **First real match-play result, stopped by hand at 70 games**: `-45
+ =0 +25` (score 0.357), llr trending toward H0 but nowhere near
+ resolved (far too few games for this SPRT's bounds). At `--st 4`,
+ the NNUE candidate searched ~3.4-3.8 plies shallower than the HCE
+ head (avg depth ~10.1-10.5 vs ~13.7-13.9) and at roughly 4.6-4.8x
+ fewer nps (~285-293k vs ~1.32-1.41M) -- yet still won 25 of 70 games
+ and held a stable ~0.30-0.36 score across the run rather than
+ eroding toward 0 as the sample grew, and ran a *tighter* tree despite
+ running under HCE-tuned pruning margins (cumulative ebf
+ ~3.8-3.97 vs. head's ~4.59-5.01). Read as: the net's per-position
+ judgment looks genuinely promising, but is not yet being given a
+ fair fight -- two known confounds stack against it simultaneously
+ (raw search-depth handicap from the speed gap, and futility/
+ delta-pruning/aspiration-window margins all tuned against HCE's
+ score distribution, not NNUE's). **Zero draws is not a red flag** --
+ expected signature of a large strength/depth mismatch (the weaker
+ side tends to lose material in sharp lines well before either side
+ would reach a balanced technical ending), not evidence anything is
+ broken.
+- **Notably, draws were exactly zero** across all 70 games -- see
+ above, this is explained by the depth/strength gap, not a bug.
+- **Quantization investigation started** (not yet implemented -- see
+ "Quantization plan" below for the concrete next steps):
+ - AVX2 build support added (`AVX2=1` GNUmakefile option, scoped
+ opt-in like `GPROF=1`, not folded into the default/`-mtune=native`
+ set -- same portability reasoning as the existing `-march=native`
+ rejection). Confirmed clang's `-O3 -ffast-math` already
+ autovectorizes both `NnueForward` and `NnueAccumAddFeature`/
+ `RemoveFeature` to SSE (4-wide) without this flag; `AVX2=1`
+ widens them to real 256-bit `ymm` ops (verified via `objdump` on
+ the linked binary, not the `.o` -- the `.o` files are LTO
+ bitcode, `objdump -d *.o` shows nothing meaningful).
+ - **AVX2 is a ~3% win for classic HCE but a measured no-op for
+ NNUE** (`bench`: 410,380 -> 411,402 nps, within noise). This is
+ a real, useful negative result: it confirms the NNUE bottleneck
+ is memory-bound (cache misses gathering rows out of the ~23MB
+ `g_rgfNnueWeight` embedding table on every accumulator
+ add/remove -- the per-*move* hot path, far more frequent than
+ the once-per-evaluated-leaf FC head), not FP throughput -- wider
+ SIMD can't help a stall waiting on a cache-line fetch.
+ - **Quantization safety-checked against the real checkpoint before
+ committing to any scale constants**, since this net trains with
+ plain (unclipped) ReLU, not Stockfish's clipped ReLU that makes
+ its own int16/int8 scheme near-lossless by construction. Loaded
+ `ckpt_epoch019.pt` directly: `ft.weight` max|.|=5.01,
+ std~=1.00; FC weights all max|.| < 0.36. Sampled 2000 trials of
+ 32 active features (a realistic full-board HalfKAv2_hm count)
+ summed through `ft.weight` + bias: worst-case accumulator
+ magnitude ~26. **Conclusion: safe to quantize this checkpoint
+ directly, no retrain needed** -- even a generous fixed-point
+ scale (256-512) leaves int16 (+-32767) comfortable headroom
+ against the realistic worst case (~6,600-13,300), nowhere near
+ saturation.
+
+## Quantization plan (not yet implemented)
+
+Priority order, chosen from the AVX2 experiment above: the embedding
+table (per-*move* cost, memory-bound) matters more than the FC head
+(per-evaluated-*leaf* cost, compute-bound and already partly
+vectorized), even though the FC head is the easier, more isolated
+change. Planned sequence:
+
+1. **Int16-quantize the embedding table first** (`g_rgfNnueWeight`,
+ `g_rgfNnueBias`) -- halves its footprint (~23MB -> ~11.5MB),
+ directly targets the measured memory-bound bottleneck via improved
+ cache hit rate on `NnueAccumAddFeature`/`RemoveFeature`'s per-move
+ gather. Fixed-point scale TBD from the 256-512 range checked safe
+ above; `NnueAccumRecompute`'s accumulation itself can likely stay
+ int32 (cheap, avoids overflow entirely) even once the table it
+ gathers from is int16.
+2. **Int8-quantize the FC head second** (`nnueeval.c`'s `NnueForward`)
+ -- isolated to one file, runs once per evaluated leaf (not per
+ move), and gets the largest *relative* SIMD win (int8 dot products
+ are 32-wide under AVX2 vs. SSE's 4-wide float32) once it's worth
+ doing at all.
+3. **New weight-file format version required.** `NnueLoadWeights`
+ currently validates magic (`"NNU1"`) + dims but has no dtype/format
+ field -- an int16/int8 export with the same dims would silently be
+ read as float32 and produce garbage, exactly the failure class the
+ existing dim-check exists to prevent. Bump to `"NNU2"` (or add an
+ explicit format word) before shipping any quantized export.
+4. **New correctness test required -- `TestNnueAccumulator` will not
+ catch quantization error.** It only compares incremental
+ maintenance against `NnueAccumRecompute`; once both sides are
+ quantized identically they still agree with each other regardless
+ of whether quantization degraded the actual eval. Need a separate
+ check: float `NnueForward` vs. quantized `NnueForward`, same
+ position, across a few hundred FENs, comparing the resulting
+ centipawn deltas against some acceptance threshold (TBD).
+5. **Side benefit, not the goal**: integer incremental accumulator
+ add/subtract is exact (no float drift), so quantizing removes both
+ `_CompareAccumToRecompute`'s 0.01f epsilon tolerance and any
+ long-move-sequence floating-point drift question entirely.
+6. Re-run `test_vs_head.sh` with `NNUE_WEIGHTS` after each step to
+ measure the real nps delta before moving to the next step -- same
+ reasoning as checking AVX2 in isolation before quantizing: don't
+ stack two speed changes in one measurement.
+7. Once NNUE's effective search depth is no longer handicapped by
+ several plies at the standard time control, re-run the
+ `match_play.py` gate for a real read on judgment quality, largely
+ decoupled from the raw-speed confound -- and only then decide
+ whether margin recalibration (see "Integration into search" above)
+ is worth doing before or after a Phase B (GPU-trained, larger net)
+ run.
+
## Training compute
- No reliable local GPU. Decided against Google Colab's notebook-session
@@ -1118,36 +1289,1404 @@ interchangeable with real Stockfish `.nnue` binaries.
existing joint-table instrumentation, and only build an independent
bitboard pass if that measurement says it's worth it.** No
speculative tactical-signals pass planned up front.
-3. MakeMove/UnmakeMove accumulator plumbing (ply-indexed king-move
- backup in `PLY_INFO`) built and proven correct/fast **with the old
- `Eval()` still live** -- prove the plumbing before any trained net
- exists to evaluate against it. **Do this on a separate git branch**,
- not on `master`, given how much live search-path code it touches;
- snapshot `head_reference` first.
-4. TWIC/lichess annotation harness built; `<TT>`/`<EGTB>`/`<REC>` tag
- semantics investigated and resolved; first large annotated training
- pool assembled (Phase 1 data, per above). **Unblocked now, independent
- of steps 1-3** -- no engine-internals risk, can start immediately.
-5. First bootstrap net trained (rented GPU), quantization-aware export
- pass built.
-6. C inference code (forward pass + quantized weight loader) written and
- validated against the Python model's output on held-out positions.
-7. Wired into search behind a toggle; margins recalibrated.
-8. `match_play.py` gate; iterate; retire old `Eval()` only after a
- convincing, repeated win.
+3. **Done, on `nnue-accumulator-plumbing`.** MakeMove/UnmakeMove
+ accumulator plumbing (ply-indexed king-move backup in `PLY_INFO`)
+ built and proven correct via `testnnue.c`'s `TestNnueAccumulator`,
+ and proven fast after finding and fixing a real ~50% throughput
+ regression (see "Status (2026-10-15)" above) -- gated behind
+ `g_fUseNnueEval` so it costs nothing with NNUE off. `head_reference`
+ snapshot already existed per the existing protocol.
+4. **Done.** TWIC annotation harness built and run (Stage A, 2.55M
+ positions from 30,000 `twic_filtered.pgn` games); `<TT>`/`<EGTB>`/
+ `<REC>`/`<=>`/`"only"` tag semantics investigated and resolved (see
+ "Status (2026-10-14)" above, including the real engine bugs found
+ along the way).
+5. **Done for Phase A (CPU-trained bootstrap, not the GPU-trained
+ Phase B net).** First net trained (`nnue_train.py`, 20 epochs,
+ train MSE 0.0065 / holdout MSE ~0.022) and exported
+ (`export_nnue_weights.py` -> `weights.nnue`). **Quantization-aware
+ export pass is explicitly NOT done yet** -- current export is
+ plain float32; see "Quantization plan" above for the concrete,
+ not-yet-started next steps.
+6. **Done**, modulo quantization. C inference code (`nnueeval.c`'s
+ `NnueForward` + `NnueLoadWeights`) written and running against real
+ search positions via the `nnue eval` command and live search
+ integration. **Not yet validated against the Python model's output
+ position-by-position** -- worth doing (a batch of FENs run through
+ both `nnue_model.py`'s `forward()` and `NnueForward`, comparing
+ scores) before trusting the C side's numerical correctness beyond
+ "it loads and produces plausible-looking scores," which is all
+ that's been checked so far.
+7. **Done.** Wired into search behind `g_fUseNnueEval`/
+ `g_fNnueWeightsLoaded` (`Eval()`'s short-circuit). **Margins NOT
+ yet recalibrated** -- still running under HCE-tuned futility/EFP/
+ LMR/NMP margins, which the first match-play result's analysis
+ flags as a likely real confound, not yet quantified.
+8. **In progress, first result in hand, inconclusive by design (stopped
+ early).** `match_play.py` gate: 70 games, `-45 =0 +25`, llr nowhere
+ near either SPRT bound. Not resumed yet -- current priority is
+ closing the speed gap (quantization) before spending more match
+ time on a comparison known to be confounded by it. Old `Eval()`
+ stays live and is the default (`g_fUseNnueEval` defaults `FALSE`)
+ until NNUE wins convincingly and consistently under this gate.
+
+## Status (2026-10-15, quantization session): both quantization steps done, validated, and measured
+
+Both steps of the "Quantization plan" above are now implemented, on
+the `nnue-accumulator-plumbing` branch, in this order:
+
+- **Step 1 -- embedding table int16, done.** `g_rgfNnueWeight`/
+ `g_rgfNnueBias` (`chess.h`/`nnueaccum.c`) are now `SHORT`, at a fixed
+ scale of 4096 (checked against `ft.weight`'s real max|.|=5.01 --
+ `5.01*4096 ~= 20502`, well under `MAX_SHORT`, ~1.6x headroom for a
+ larger Phase B net). `NNUE_ACCUMULATOR.rgiValue` (was `rgfValue`) is
+ `INT` (int32), not `SHORT` -- summing up to 32 active features'
+ `SHORT` rows can exceed `SHORT`'s range even though each individual
+ row fits (measured worst-case real accumulator magnitude ~26 in
+ float-equivalent units, i.e. ~106,500 at this scale -- 3x past
+ `MAX_SHORT`). `NnueForward` converts back to float (divides by
+ `g_fNnueFtScale`) only where the accumulator is read, never where
+ it's maintained -- incremental `MakeMove`/`UnmakeMove` updates stay
+ purely additive integer arithmetic, exact, no float drift.
+ `testnnue.c`'s `TestNnueAccumulator` updated to match (integer test
+ weights, exact-equality comparison instead of an epsilon tolerance --
+ a real correctness upgrade this dtype change enabled, not just a
+ mechanical port). Passes clean under `DEBUG=1 TEST=1`.
+- **Step 2 -- FC head int8, done.** `g_rgfNnueFc1Weight`/
+ `g_rgfNnueFc2Weight`/`g_rgfNnueFc3Weight` (`chess.h`/`nnueeval.c`)
+ are `CHAR` (int8, signed), one weight scale per layer derived
+ exactly from that layer's own max|.| (no sampling needed, weights
+ are fixed and known). Biases stay `float32` (tiny arrays, added
+ post-dequant, not worth quantizing). Activations are UCHAR (uint8,
+ unsigned -- confirmed all three stages are strictly post-ReLU,
+ `min=0.0000` on a real 50k-position sample, so no bits wasted on a
+ sign). `NnueForward` rewritten stage-by-stage: dequantize accumulator
+ -> ReLU -> requantize to UCHAR at AS1 -> int8x8 dot product
+ accumulated in `INT` -> dequantize (divide by `AS*WS`) + bias -> ReLU
+ -> requantize at AS2 -> repeat through fc2/fc3. Matches a Python
+ numpy simulation of the identical quantized math, built and checked
+ before touching any C code (see below).
+ - **Activation scales (AS1/AS2/AS3) are calibrated from real
+ sampled positions, not a theoretical bound** -- unlike weights,
+ activation ranges aren't knowable from the checkpoint alone.
+ `export_nnue_weights.py`'s `CalibrateActivationScales` runs
+ 50,000 real positions from `tensors_train/` through the float
+ model with forward hooks, takes the observed max per stage, and
+ applies a 1.3x headroom multiplier before deriving the scale
+ (`AS = 255 / (observed_max * 1.3)`). Real observed maxes:
+ `fc1_in` 27.12, `fc2_in` 19.11, `fc3_in` 7.72 (p99.9 well below
+ max in each case -- confirms the max is tail-driven, headroom is
+ doing real work, not padding noise).
+ - **File format bumped to `"NNU3"`** (`NnueLoadWeights` rejects
+ both older `"NNU1"` and `"NNU2"` files explicitly, not silently).
+ Header now carries `ft_scale` (1x float), `act_scale` (3x float:
+ AS1/AS2/AS3), `w_scale` (3x float: WS1/WS2/WS3) before the
+ weight arrays themselves.
+ - **Accuracy measured before touching any C code, same discipline
+ as step 1's safety check**: a numpy simulation of the exact
+ quantized int8 math (uint8 activations, int8 weights, int32
+ accumulation, per-layer scales) against the real float model,
+ 5000 real holdout positions: **mean |delta| 1.55cp, median
+ 0.99cp, max 20.91cp, zero clipping anywhere** (weights or
+ activations, all three layers, out of 2.56M+160K+1K+32 values
+ checked). Context: the project's own `sd10`-vs-`sd16`
+ depth-adequacy check (training-label noise already tolerated)
+ measured mean |delta| 19cp -- this quantization's error is
+ **over 10x smaller** than noise already baked into the training
+ data. Confirmed again directly in C after implementation: `nnue
+ eval` on 4 hand-picked real FENs vs the Python float reference,
+ deltas of 1/2/11/0 cp -- consistent with the offline simulation,
+ no sign of an implementation bug.
+- **AVX2+FMA confirmed as a real, separate win once real weights are
+ loaded** (not the "measured no-op for NNUE" result from the
+ pre-quantization investigation above -- that measurement predated
+ knowing which function actually dominated runtime, see the
+ profiling finding below). `gmake AVX2=1`: `objdump` on the linked
+ binary confirms `NnueForward`'s dot-product loops widen from 4-wide
+ `xmm`/`mulps`+`addps` to 8-wide `ymm` with 99 real `vfmadd*` fused
+ instructions. Measured: int16-table-only build went from ~381-384k
+ nps (no `AVX2=1`) to ~409-410k nps (`AVX2=1`) -- a real, repeatable
+ ~7% win, zero retrain, zero risk.
+- **Profiling (`docs/PROFILE.md`'s FreeBSD `pmcstat -P` + `gprof` path)
+ found `NnueForward` -- the FC head, not the embedding table --
+ dominates runtime: 34.0% of total self-time, alone**, more than
+ double the next entry (`SlidePiece`, 12.6%) and roughly on par with
+ *all* accumulator-maintenance-related functions combined even when
+ generously over-attributing their non-NNUE move-bookkeeping cost to
+ NNUE. `NnueAccumAddFeature`/`RemoveFeature` don't even appear as
+ separate flat-profile line items -- small enough to get inlined into
+ their `SlidePiece`-family callers. This directly revised the
+ quantization plan's priority ordering: the embedding table (step 1)
+ was assumed memory-bound-dominant from the AVX2 experiment's
+ "no-op for NNUE" result, but that result just meant AVX2 doesn't
+ help *that* code path -- it didn't establish the table was the
+ actual bottleneck. `fc1`'s matmul (`2*L1 x L2 = 2*256*32 = 16,384`
+ multiply-adds per evaluated leaf) being far larger than `fc2`
+ (1,024) or `fc3` (32) explains why: cost at each layer boundary
+ scales with the *product* of adjacent layer widths, and the
+ 512-wide-to-32 transition is the one expensive step in this
+ topology, more frequently hit (once per evaluated leaf) and more
+ expensive per-call than the table gather (once per move, but only
+ a handful of 256-element row reads).
+- **Net bench result, chaining every change in this session**:
+
+ | Build state | nps (real weights, `nnue on`) |
+ |---|---|
+ | Float32 everywhere (pre-session baseline) | ~411,000 |
+ | int16 table, no `AVX2=1` | ~381,000-384,000 |
+ | int16 table + `AVX2=1` | ~409,000-410,000 |
+ | int16 table + int8 FC head + `AVX2=1` | **~559,000-563,000** |
+
+ Net effect: **~37% faster** than the pre-session float32 baseline,
+ closing the gap to classic HCE (~1.51M nps) from 3.9x down to 2.7x.
+ Correctness verified at every step (self-tests, offline Python
+ simulations before touching C, then direct C-vs-Python spot checks
+ after).
+- **`WS1`/`WS2`/`WS3` derivation note**: unlike the embedding table
+ (one global scale, since it's one homogeneous array), each FC layer
+ gets its *own* weight scale, derived from that layer's own max|.| --
+ `fc1.weight` max 0.350, `fc2.weight` max 0.361, `fc3.weight` max
+ 0.334 (all much smaller than `ft.weight`'s 5.01, but the relevant
+ comparison for int8 resolution isn't the max, it's the *step size*
+ relative to each layer's std -- at full-range scale this works out
+ to roughly 18 quantization levels per standard deviation for the FC
+ weights, versus thousands for the int16 table. The 1.55cp measured
+ result shows this coarser per-weight resolution doesn't translate
+ 1:1 into coarser *output* accuracy -- dot-product summation over
+ many terms (512/32/32) partially cancels independent rounding
+ errors, and ReLU's sparsity (46-69% exact zeros measured on real
+ activations) contributes zero error for free on a large fraction of
+ terms.
+
+## Lazy-evaluation tiering plan (started 2026-10-15, not yet wired into search)
+
+Classic `Eval()` has three real exits -- super-lazy (material only),
+lazy (material + a defined subset of positional terms), full -- each
+gated by a margin derived from a *measured population* of real
+positions, not a modeled/assumed distribution. `Eval()`'s current NNUE
+short-circuit (added when NNUE was first wired into search, see
+"Status (2026-10-15): training run finished" above) bypasses all of
+this -- it's an unconditional full `NnueForward()` call, with a comment
+explicitly noting the margin machinery below it "has not been
+recalibrated against NNUE's score distribution yet." This section
+plans closing that gap.
+
+**The mechanism, read directly out of the live code** (not
+reconstructed from memory -- see `eval.c`'s
+`iSuperLazyResidualPessimisticByOwnArmy`/`OptimisticByOwnArmy`/
+`P50ByOwnArmy` tables and their use in `Eval()`'s super-lazy exit,
+~line 7150):
+
+- Every one of these tables is **literally empirical percentiles read
+ off a measured population of real positions** -- the comment blocks
+ above several of them show raw population stats before the table
+ (e.g. `n=104187433 min=-68 p1=-10 ... p50=10 ... max=165`). No
+ distribution shape is assumed; the percentile is read directly off
+ sorted real data, per material bucket (`_MaterialBucket`/per-side
+ army-scaler bucket, 8 buckets in the existing tables).
+- **Two different statistics for two different jobs, not one margin**:
+ - **p50 is a bias correction** -- the typical/unbiased value fed to
+ `iPositional` (a lazy exit's best-guess estimate of the missing
+ positional terms, consumed downstream by e.g. QSearch's own
+ futility margins). Current NNUE short-circuit reports a flat
+ `{0, 0}` here -- the "intentionally least-specific but safe"
+ placeholder, not a real estimate. A calibrated p50 of the probe's
+ own residual could improve this once real residual data exists.
+ - **p2/p98 (not p5/p95) is the soundness bound** -- answers "how far
+ could the true full-eval score plausibly be from this cheap
+ estimate, in the realistic tail," and is what actually gates
+ whether the lazy exit is safe to take. The exact comment: *"p50 is
+ the right statistic for a bias correction, p2/p98 is the right
+ statistic for a soundness bound."*
+ - Direction is asymmetric and combines two terms before a `MAX0()`
+ floor: checking against `alpha` (could this fail low?) uses the
+ **optimistic** tail (most generous plausible correction in the
+ mover's favor); checking against `beta` (could this fail high?)
+ uses the **pessimistic** tail (least favorable plausible
+ correction). Getting this sign/direction wrong would silently
+ invert the safety guarantee -- needs care, not pattern-matching,
+ when implemented for real.
+- This two-statistic pattern (p50 bias, p2/p98 safety bound) is what
+ the NNUE lazy tier should reuse, not a new invention -- same
+ methodology the existing calibration harness
+ (`CALIBRATE_MARGIN_SAFETY`, `RecordSuperLazyMarginSafetySwing`, swing
+ histograms bucketed by material) already proved out for classic
+ `Eval()`'s own tiers.
+
+**The proposed NNUE lazy-tier signal**: not a second, separately
+trained net (that reintroduces the calibration-mismatch problem
+already rejected in the "Architecture decisions" section above) --
+instead, a **linear probe fit directly on the real net's own
+accumulator** (`fc1`'s input: the ReLU'd, concatenated 512-wide
+accumulator halves), which is already computed and resident at every
+node for free as a side effect of `nnueaccum.c`'s incremental
+maintenance, before `NnueForward`'s expensive `fc1->fc2->fc3` pass
+(34% of runtime, see profiling finding above) ever runs.
+
+**Progress so far**:
+- **Offline probe fit, done, preliminary.** Ridge regression (512
+ features + bias, lambda=1.0) fit against 80,000 real positions from
+ `tensors_train/`, targeting the real float net's own raw output
+ (`forward()`, pre-sigmoid). Saved to
+ `annotate_pgns/probe_weight.npy`/`probe_bias.npy`.
+- **Measured residual (full net score minus probe score), 20,000 held-out
+ positions from the same sample**: mean |residual| 50.10cp, median
+ 33.09cp, p2=-166.79cp, p50=0.33cp, p98=+169.14cp. **An order of
+ magnitude coarser than the int8 quantization's 1.55cp** -- expected,
+ since a flat linear projection discards all of `fc1->fc2->fc3`'s
+ nonlinear structure, unlike int8 which preserves the same
+ computation at lower numeric precision. This sets real expectations:
+ the implied safety margin (~150-200cp) is large relative to typical
+ eval swings, so the lazy exit will only fire on nodes already
+ lopsided relative to the search window -- a real but likely narrower
+ slice of total nodes than hoped going in.
+- **Real fit-vs-validation gap observed** (mean 37.85cp on the fit
+ split vs 50.10cp held out, p98 153cp vs 195cp) -- larger than
+ expected given 80k examples against 512 features plus ridge
+ regularization. Not yet root-caused (could be regularization
+ strength not cross-validated, could be real limits of a flat linear
+ fit over this representation, could be this offline sample's size).
+ **Flagged as unresolved, not yet trusted as final** -- this was an
+ offline, single-position-sample check, explicitly a preliminary
+ step, not the real calibration.
+
+**Not yet done -- the actual next steps, in order**:
+1. Root-cause the fit/validation gap above (cross-validate `lambda`,
+ or just use more fitting data) before trusting any residual
+ numbers from this probe as final.
+2. **Build the real calibration pass**: a `CALIBRATE_MARGIN_SAFETY`-
+ style harness for the NNUE lazy probe specifically -- same pattern
+ as classic `Eval()`'s existing one (run the full `NnueForward` pass
+ anyway even when the probe-based exit would fire, record the swing,
+ restore the real build's behavior before returning) -- against
+ **real search-node traffic**, not another offline position sample.
+ This is the measurement that actually matters; the offline check
+ above was only ever a cheap sanity gate before investing in this.
+3. From that real population, derive p50 (bias correction) and p2/p98
+ (safety bound) -- possibly bucketed by material/phase the way
+ HCE's own tables are, possibly not. **Whether NNUE's accumulator
+ already implicitly encodes phase** (unlike HCE's explicitly
+ material-gated lazy terms) **is an open, empirically-answerable
+ question from this same population** -- check whether the residual
+ distribution actually shifts meaningfully bucket-to-bucket before
+ committing to that complexity.
+4. Wire the gate into `Eval()`'s existing NNUE short-circuit, using
+ `iAlpha`/`iBeta` already passed into `Eval()` -- no new interface
+ needed, this is a parallel early-return block next to the current
+ unconditional bypass.
+5. Ship it instrumented-only first (swing recorded, real build
+ behavior unaffected) before trusting it live, same rollout
+ discipline as every other change in this project.
+
+The super-lazy (material-only, no net at all) idea from the
+"Architecture decisions" section above remains separate, more
+tentative, and explicitly not decided either way -- the calibration
+risk there (no shared units with the WDL-sigmoid-calibrated net
+output) is structurally different from this linear-probe tier, which
+shares the real net's own trained representation and units by
+construction.
+
+## Status (2026-10-16): lazy probe gate shipped live, QSearch futility margins fixed, same-depth ECM win confirmed, overnight match queued
+
+Picking up from the "Lazy-evaluation tiering plan" section above --
+the fit-vs-validation gap was root-caused as pure sampling noise (more
+data + the real `tensors_holdout/` split made it vanish; lambda barely
+mattered, the model was never near overfitting), and everything from
+"build the real calibration pass" onward is now done.
+
+- **Fit/validation gap resolved**: 500k-example fit + real `tensors_
+ holdout/` evaluation (not another ad-hoc split) showed holdout
+ *better* than fit (38.68cp vs 41.09cp mean) -- the earlier 50.10cp
+ number was noise from a too-small single sample, not overfitting.
+- **Real calibration done properly, twice**: first pass used the three
+ curated tactical suites (91M samples) -- too narrow a slice (hand-
+ picked for sharp tactics), confirmed by a second pass against
+ `tests/cross_sectional_200k.ep_` (200,000 diverse positions, 16
+ parallel single-threaded shards, `sd 6`, 3.09 BILLION real search-
+ node samples) landing close to but meaningfully more trustworthy
+ than the first (p98 margins `[180, 200, 190, 180, 170, 160, 150,
+ 150]` by material bucket vs the tactical-suite-only `[190, 200, 200,
+ 190, 190, 170, 150, 150]` -- also surfaced real tail outliers (max
+ 534cp) the narrower sample never saw).
+- **A real, separate bug found and fixed along the way**: chaining
+ multiple commands after `script <file>` in one `--batch --command`
+ string (e.g. `script a.ep_; script b.ep_; calibrate marginsafety`)
+ silently truncates every search except the last to a shallower depth
+ than requested -- `PushNewInput` (`input.c`) splits and queues
+ *everything* up front, and `searchsup.c`'s mid-search
+ `NumberOfPendingInputEvents()` check (legitimate for live xboard/
+ WinBoard responsiveness) fires on that queued-but-irrelevant trailing
+ text, treating it as "stop, something wants my attention." Root
+ cause confirmed by direct repro (`sd 10` silently landing at depth 8,
+ 8x fewer nodes) -- **not fixed in the engine** (the interactive-
+ responsiveness behavior is legitimate and deliberately left alone;
+ the input-queueing semantics are too sensitive to touch for a
+ testing-only problem that already has a zero-risk workaround).
+ **Workaround, now the required pattern for any multi-step batch
+ calibration**: never chain unrelated commands after `script` in one
+ `--command` string; if a command needs to run after a suite finishes
+ in the same process (e.g. `calibrate marginsafety`), put it as its
+ own line *inside* the `.ep_` file instead -- `script.c`'s own
+ `_ProcessSuiteLine` already dispatches any non-"go" line via the
+ queue-free `ExecuteCommandLineDirectly`, so it runs in-order with no
+ truncation risk. For N-way sharded runs (per CLAUDE.md's existing
+ convention), each shard gets its own single `script` call + its own
+ `calibrate marginsafety <path>` line inside the file, in its own
+ process -- then combine the N raw histogram files after the fact.
+- **The "super lazy" (material-only) tier was investigated and
+ rejected as a live feature** -- `pos->iMaterialBalance[color]` is
+ already in the same cp units as `NnueForward`'s output
+ (`VALUE_PAWN=100`), so no fit/regression was needed at all (unlike
+ the accumulator probe) -- straight subtraction, same harness. Real
+ measured residual against the full net, same 3.09B-sample
+ cross-sectional pool: mean 124-344cp, p98 510-1230cp by bucket --
+ **6-8x larger than the accumulator probe's** (47-77cp mean, 150-200cp
+ p98). Shape is an inverted-U by material bucket (peaks in the
+ middle, smaller at both bare-king endgames and near-full-material
+ positions), not the simple monotonic-opposite-of-the-probe trend
+ originally guessed. Conclusion: material alone is too crude a proxy
+ for *this* net's output to fire often enough to be worth it -- NNUE's
+ accumulator captures far more positional judgment relative to
+ material than a handcrafted eval does, unlike HCE where material
+ really is close to the final answer. Not built as a live gate.
+- **Lazy probe gate shipped live** (`nnue lazyprobeon`/`lazyprobeoff`,
+ defaults `FALSE` same as `g_fUseNnueEval`): `Eval()`'s NNUE short-
+ circuit now actually skips `NnueForward` when
+ `NnueProbeScore() +/- margin` confidently resolves fail-high/fail-low
+ against `[iAlpha, iBeta]`, dual-mode via `#ifdef CALIBRATE_MARGIN_
+ SAFETY` (calibration build always still runs the full pass for
+ measurement; release build actually skips). Held-out validation
+ (`ecm_confident_sample2.ep_`, not used for calibration) showed
+ exceeded rates mostly at or under the ~2% the p98 choice implies
+ (buckets 2-7: 0.87-2.24%); buckets 0-1 ran hotter (3.47-4.67%,
+ consistent with those being the smallest-sample buckets in
+ calibration, so the noisiest estimates).
+- **Measured lazy-probe exit rate on real search traffic** (`twic_
+ sample.ep_`, `sd 6`): **14-23% of NNUE eval calls resolve via the
+ cheap probe**, varying by material bucket (peak ~20-23% in buckets
+ 3-5, low 8.7% in bucket 7/near-full-material). Most calls still pay
+ for the full forward pass.
+- **Real, clean, sequential (not parallel -- see below) same-depth
+ measurement, `ecm_hard_quick`, `sd 10`, gate off vs on**:
+
+ | | OFF | ON |
+ |---|---|---|
+ | Solved | 32/90 | 33/90 |
+ | Nodes | 47,790,608 | 50,438,386 |
+ | Time | 112.4s | 107.1s |
+ | nps | 425,271 | 470,938 |
+
+ **~10.7% real per-node throughput win, no solve-rate regression**
+ (if anything +1). Note nodes went *up* while time went *down* --
+ the gate changes which nodes get pruned (approximate score at some
+ nodes shifts alpha-beta cutoffs), so node count isn't held constant
+ the way `sd` depth is; nps is the cleaner isolated signal.
+ **Caveat on parallel-run timing noise, confirmed twice this
+ session**: running gate-off and gate-on as simultaneous background
+ jobs on a shared box can be contaminated by real external load (cron
+ jobs, rsyncs, other users) even when each job's own `--cpus 1` leaves
+ plenty of idle capacity -- the fix was a clean *sequential* run on an
+ otherwise-idle box, not avoiding parallelism itself (6-way `--cpus 1`
+ parallel jobs were confirmed fine once the box was actually quiet).
+- **Full three-suite combined result, same clean methodology**:
+ 108/191 (56.5%) solved OFF vs 112/191 (58.6%) ON, total time 247.7s
+ vs 240.3s (-3.0%) -- consistent with the single-suite sequential
+ number, no regression anywhere.
+- **Independently, at the SAME fixed `sd 10` depth (no lazy gate,
+ separate from all of the above): NNUE solves substantially more ECM
+ tactics than classic HCE** -- 108/191 (56.5%) vs HCE's 72/191
+ (37.7%) across all three curated suites. Since depth is held
+ constant, this isn't explained by NNUE's remaining speed deficit.
+ **Correction, 2026-10-17: do not read this as evidence the net
+ plays better chess.** Every real head-to-head game match run
+ since (see "Status (2026-10-17)" below -- five independent
+ configurations: two different nets, `st 4` and `sd 10`, with and
+ without the lazy probe) has NNUE losing decisively to classic HCE,
+ including at matched search depth where this ECM result would
+ predict an NNUE edge. ECM is a narrow slice (forced-tactic/mate
+ puzzles, where qsearch/extensions/move-ordering machinery shared
+ by both evaluators does most of the work) and evidently does not
+ generalize to real-game strength. As of this writing, solving more
+ ECM puzzles than HCE is the *only* thing either NNUE net has been
+ shown to do better than HCE -- it has not won a single head-to-head
+ match, at any depth or time control tried.
+- **QSearch futility-pruning margins needed their own investigation,
+ separate from the eval-exit-gate margins above** -- `PLY_INFO.
+ uLastPositionalSource` (the per-ply tag `search.c`'s
+ `FUTILITY_BASE_MARGIN_BY_SOURCE[]` reads to decide how much slack to
+ give QSearch's delta/futility pruning) was **never being set at all**
+ by either NNUE exit -- every NNUE-sourced score was silently
+ inheriting a stale value from the last classic-`Eval()` call at that
+ ply slot (or the C zero-init default, `EVAL_POSITIONAL_SOURCE_FULL`,
+ if none ever ran). Fixed: added `EVAL_POSITIONAL_SOURCE_NNUE_FULL`/
+ `NNUE_LAZY`, set at both NNUE return sites, each with its own
+ `FUTILITY_BASE_MARGIN` entry and always-on exit-rate counters
+ (`calibrate exitrate` extended to report NNUE's own lazy-vs-full
+ split, separate from the eval-exit gate's).
+ - **First real measurement** (`twic_sample.ep_`, 1500 positions,
+ `sd 6`, the same pool/depth the *original* classic margins were
+ derived from): `nnue full`'s surprise rate was 7.41% in the
+ checking-capture/promo gate at the 450 starting placeholder --
+ well above anything in classic's documented history. Widening to
+ 650 tested as a complete no-op (identical 7.41%, identical `n`).
+ - **Root-caused, not just observed**: added a direct diagnostic
+ (raw pre-clamp `iFutility` value, before `MAX0`) and confirmed
+ `nnue full`'s population is **100.00% clamped to the `MAX0`
+ floor** (mean pre-clamp -589.3) -- the margin constant is never
+ actually read for this population, so no value could have shown
+ an effect. Classic `full eval`, measured the identical way on
+ the identical pool, is clamped 91.05% of the time (not 100%,
+ and less deeply negative: mean -282.5) -- saturation is a real,
+ general QSearch structural property (not NNUE-specific: classic
+ `regular lazy`'s 24.95% clamp rate matches `nnue lazy probe`'s
+ 24.76% almost exactly), but NNUE's `FULL` tier saturates somewhat
+ more completely than classic's.
+ - **Surprise magnitude (not just rate) measured too**, resolving
+ a live "is NNUE just noisier" vs "does NNUE miss bigger things"
+ question: `nnue full`'s surprises, when they happen, run
+ *smaller* than classic `full eval`'s (mean 477.5cp vs 938.7cp,
+ `>100cp` 9.88% vs 28.74%) -- same pattern for `nnue lazy probe`
+ vs classic `regular lazy` (5085.3cp vs 11030.2cp mean; 19.45% vs
+ 69.73% `>100cp`). So the net effect is: NNUE_FULL fails *more
+ often* but *less expensively* each time -- consistent with more
+ frequent small eval jitter near the decision boundary, not with
+ NNUE being tactically blind to bigger things than HCE already
+ is. One sample verified by hand (a genuine `Qh7+` mate-in-1,
+ confirmed independently via a real `sd 16` search) to make sure
+ the diagnostic's rare huge-magnitude entries were real finds, not
+ a measurement bug.
+ - **Decision: `FUTILITY_BASE_MARGIN_NNUE_FULL` reverted to 450**
+ (matching classic `FULL`) rather than kept at the untested 650 --
+ the saturation finding means neither value was actually
+ validated by this data, and 450 is the more defensible default
+ given NNUE_FULL's measured surprises skew smaller than HCE's own.
+ **`FUTILITY_BASE_MARGIN_NNUE_LAZY=200` kept** -- this one *is*
+ genuinely validated (only 24.76% clamped, moved the expected
+ direction when tightened from 350, landed in a sane 1.3-2.1%
+ surprise-rate range, smaller surprise magnitudes than classic's
+ own `regular lazy`).
+ - **TODO, not yet done**: find and calibrate against the actual
+ non-saturated `NNUE_FULL` population specifically (the ~0% slice
+ this test pool never surfaced) before trusting any `NNUE_FULL`
+ margin as real, NNUE-specific-validated rather than a reasonable
+ borrowed default. Likely candidate: nodes where `iEval` sits
+ close to `iAlpha` rather than comfortably above it (the
+ saturated case is "`iEval` way above `iAlpha`" -- look at the
+ complement).
+- **Match-play CLI wiring added, needed before any NNUE+lazy-gate
+ match can run at all**: `--nnueweights` already loaded weights and
+ set `g_fUseNnueEval=TRUE` at startup (for `match_play.py`'s fresh-
+ process-per-game spawning, no interactive commands available) --
+ but there was no equivalent for the probe/lazy gate at all. Added
+ `--nnueprobeweights <file>` (same convention, sets
+ `g_fUseNnueLazyProbe=TRUE` automatically), and extended `eval_tune/
+ test_vs_head.sh`'s `NNUE_WEIGHTS` wrapper with a new
+ `NNUE_PROBE_WEIGHTS` env var that injects `--nnueprobeweights` into
+ the same shim. Verified working end-to-end (both flags load and
+ enable correctly) before trusting it for an unattended run.
+ **Gotcha**: `test_vs_head.sh`'s default (no candidate path given)
+ rebuilds the candidate itself via `gmake clean && gmake release
+ MP=1 PERF_COUNTERS=1` -- **without `AVX2=1`** -- silently losing the
+ ~7% AVX2/FMA win if not passed a pre-built candidate path instead.
+- **Overnight `match_play.py` run queued** (not yet run/reported) --
+ NNUE+lazy-gate candidate vs classic HCE `head_reference`, using the
+ new `NNUE_WEIGHTS`/`NNUE_PROBE_WEIGHTS` env vars and an explicit
+ pre-built `AVX2=1` candidate binary to avoid the gotcha above.
+
+## Next up
+
+1. Read the overnight match result once it's in -- first real
+ judgment-quality read with both the quantization work and the lazy
+ probe gate live together, at a real time control rather than fixed
+ `sd` depth (so the remaining ~2.7-3.2x speed gap is a live confound
+ again, same as the original 70-game result, but now with a smaller
+ gap and the added lazy-gate throughput).
+2. The `NNUE_FULL` futility-margin TODO above -- find the non-saturated
+ population and calibrate against it specifically, rather than
+ continuing to rely on a borrowed classic-HCE default.
+3. Decide whether the ~3-11% nps win from the lazy probe gate
+ (measured) is worth the complexity on its own, or whether it's more
+ valuable combined with further quantization work on the FC head's
+ remaining float32 pieces (biases) or other speed levers -- revisit
+ once the overnight match gives a real strength read to weigh
+ against.
+
+## Status (2026-10-17): material-blindness fixed and verified, but zero
+## head-to-head wins against classic HCE in five independent trials
+
+**Summary, stated plainly up front**: a real, measured defect (the net
+could not distinguish losing a pawn from losing a queen in crowded
+midgame positions) was found and fixed this session. The fix is
+verified to work, at the population level, with a new reusable tool.
+But **no NNUE configuration tried today -- old net, new net, with or
+without the lazy probe, at `st 4` or at matched `sd 10` depth -- won
+or even drew competitively against classic HCE.** The only thing
+either net has been shown to do better than HCE is solve more ECM
+puzzles at matched depth (see the correction just above). Do not
+read anything below as evidence the net plays winning chess yet.
+
+### The material-blindness bug, found and fixed
+
+- **Found via `sanity_check_net.py`** (new tool, `annotate_pgns/`):
+ loads a raw training checkpoint directly (no export/C round-trip
+ needed -- the earliest point in the pipeline a net can be checked)
+ and removes one piece at a time from hand-picked and (better) real
+ sampled positions, reporting the score delta. On the Phase A net
+ trained with plain `λ=1.0` MSE against WDL-sigmoid targets: removing
+ a pawn, a knight, a rook, or a queen from the same midgame position
+ produced nearly indistinguishable score drops (-0.56 to -0.70) --
+ directionally correct, magnitude-blind.
+- **Root cause, two compounding mechanisms, neither one specific to
+ which corpus a position came from**:
+ 1. Sigmoid-gradient argument (later found incomplete on its own --
+ see below): training loss is MSE against `sigmoid(raw/K)`, and
+ sigmoid's gradient vanishes once a position is already clearly
+ lost/won.
+ 2. **Feature dilution / shortcut learning in crowded positions**
+ (the more complete explanation): with ~20+ active embedding
+ features summed per position, a net can match training labels
+ well without correctly attributing any single piece's marginal
+ value, because real games never present a controlled
+ single-piece-ablation pair to force that attribution. Confirmed
+ empirically: the bug was severe in crowded midgame positions and
+ essentially absent in sparse endgames (2-6 pieces), where each
+ piece dominates a much larger share of the accumulator.
+- **Data rebalancing (TWIC downsampled to match self-play 1:1 by game
+ count) was necessary but not sufficient.** It measurably helped the
+ sparse-endgame case (which TWIC's human-resignation bias
+ structurally starves) but did nothing for the crowded-midgame case,
+ confirming the bug was a representation/optimization problem, not
+ purely a data-coverage gap.
+- **Two cheap levers tried and only partially effective on their
+ own**:
+ - `loss_exponent=2.6` (Stockfish's own documented value, reweights
+ the loss toward large errors): modest effect, spread narrowed but
+ monotonicity still failed, bishop/rook order still inconsistent
+ across test positions.
+ - λ-sweep (blend in `r_outcome`, game result): **made things
+ actively worse** -- λ=0.5/0.0 flipped the sign of several
+ deltas (removing a piece appeared to *help* the score). See the
+ "λ-sweep methodology was weaker than later standards" note below
+ before trusting this conclusion, though -- it has not been
+ properly re-tested since.
+ - **Lesson from this part of the investigation**: a single
+ hand-picked test position cannot reliably settle a question like
+ "does the net value a bishop more than a rook" -- one test showed
+ bishop > rook, a different hand-picked position showed the
+ reverse. This is per-position confounding (an active bishop vs.
+ a boxed-in rook on that specific square/position), not
+ necessarily evidence about the net's general prior.
+- **Fix that worked: an auxiliary material-probe loss**
+ (`nnue_model.py`'s `material_probe`, a plain `Linear(512, 1)`
+ reading the same concatenated accumulator `fc1` does). Trained
+ jointly with the main WDL loss: `loss = wdl_loss +
+ material_loss_weight * mse(material_probe(accumulator),
+ material_balance)`, where `material_balance` is computed directly
+ from the FEN (pawn=1..queen=9, mover-relative, no sigmoid --
+ deliberately non-saturating). `material_loss_weight=0.01` chosen to
+ keep the material term's raw contribution smaller than the WDL
+ loss's throughout training (per the user's explicit concern about
+ over-fixating on material, HCE-style) -- confirmed: material loss
+ contributed ~0.019 vs. WDL's ~0.044 at epoch 0, ~0.007 vs. ~0.022 at
+ epoch 19.
+ - **Mechanistically why this is safer than adding material to the
+ output directly**: the probe loss only updates `material_probe`
+ itself and (via backprop) the shared `ft` embedding table -- it
+ never touches `fc1`/`fc2`/`fc3`, the actual eval head. The eval
+ head stays free to deviate from pure material counting for
+ positional reasons; only the *representation underneath it* is
+ forced to keep material linearly recoverable.
+ - Needed more epochs than the baseline run to show its full effect
+ -- material-probe MSE was still falling at epoch 19 (not
+ plateaued), extending to epoch 34 took it from a single hand-picked
+ test's queen delta of -0.815 (already beating rook's -0.727) to
+ a much larger, cleaner separation.
+
+### `sanity_check_net.py --cross-sectional`: the real validation tool
+
+Single hand-picked positions were repeatedly shown to be unreliable
+(see above) -- replaced with a population-level check: sample N real
+positions from a holdout pool, remove one random piece of each
+present type per position, pool deltas by piece type across the whole
+sample, report mean/median/std/p25/p75. On the matprobe net
+(2000 sampled holdout positions): **population means came out cleanly
+monotonic in piece value, both directions** -- pawn -0.747/+0.786,
+knight -1.363/+1.581, bishop -1.500/+1.710, rook -2.483/+2.907, queen
+-4.160/+5.024 (own-piece-removed / opponent-piece-removed). The
+earlier bishop-vs-rook inconsistency vanished entirely at the
+population level, confirming it was per-position noise, not a real
+defect. Scaling is compressed relative to nominal chess piece values
+(queen/pawn ratio ~5.6x here vs. nominal 9x) -- expected and likely
+correct, not a bug, since training targets are WDL-sigmoid-calibrated
+scores, not raw material counts, and a large material edge's marginal
+effect on win probability is genuinely smaller than its point value.
+
+This tool (`annotate_pgns/sanity_check_net.py`) should be run on any
+future checkpoint before trusting a single hand-picked-position read
+-- `--cross-sectional --n-positions 2000` against a holdout pool,
+`python3 sanity_check_net.py --ckpt <path>` for the quick hand-picked
+smoke test.
+
+### Export and C-side validation: clean
+
+`export_nnue_weights.py` (already quantizing directly to the `NNU3`
+format, no separate float32-only export step in the current pipeline)
+ran against the matprobe checkpoint with zero code changes needed --
+it only reads `ft`/`fc1`/`fc2`/`fc3` by name, so the auxiliary
+`material_probe` layer is silently and correctly ignored at export
+time. 6 real holdout FENs spot-checked, Python float vs. C
+int16/int8-quantized: deltas of 0.3-4.7cp, same ballpark as the
+1.55cp-mean/20.91cp-max already measured and accepted for the first
+net's quantization step. No sign/implementation bug found.
+
+### Five independent head-to-head trials, all losses
+
+All via `eval_tune/test_vs_head.sh` with `NNUE_WEIGHTS` set (and, for
+two of the five, `NNUE_PROBE_WEIGHTS` pointing at the OLD net's lazy
+probe -- see the stale-probe caveat below), against the classic-HCE
+`head_reference`-equivalent head binary:
+
+| # | net | time control | result (at stop) | llr | candidate depth vs head |
+|---|---|---|---|---|---|
+| 1 | matprobe (new) | `st 4` | -27 =2 +1 (n=30), score 0.067 | -2.07 | 12.16 vs 15.08 |
+| 2 | v3 (old) | `st 4` | -29 =1 +0 (n=30), score 0.017 | -13.05 | 10.74 vs 13.51 |
+| 3 | v3 (old) | `sd 10` | -28 =2 +0 (n=30), score 0.067 | -1.52 | 9.57 vs 9.23 |
+| 4 | matprobe (new) | `sd 10` | -24 =4 +2 (n=30), score 0.133 | -0.97 | 9.81 vs 9.70 |
+| 5 | v3 (old) + lazy probe | `sd 10` | -30 =0 +0 (n=30), score 0.000 | (total wipeout) | 9.63 vs 9.25 |
+
+**Key findings**:
+- **The new (matprobe) net consistently loses less badly than the old
+ net, under every condition tested** (trial 1 vs 2 at `st 4`: score
+ 0.067 vs 0.017; trial 4 vs 3 at `sd 10`: score 0.133 vs 0.067). The
+ material fix is a real, reproducible improvement -- just nowhere
+ near enough to close the gap with HCE.
+- **At matched `sd 10` depth (trials 3 and 4), NNUE still loses
+ decisively** -- this directly contradicts the working hypothesis
+ (built from the ECM result above) that the real-time losses were
+ purely a depth/speed-handicap artifact. They are not purely that.
+ Matching the final depth number does not mean matching search
+ quality throughout the tree -- EFP/LMR/NMP/futility margins are
+ still HCE-tuned in every one of these trials (see "Integration into
+ search" above, still not done) and could still be causing bad
+ cutoffs specific to NNUE's score distribution even when the
+ reported depth matches.
+- **The lazy probe used in trial 5 is STALE and the result should be
+ discounted**: `annotate_pgns/nnue_probe.bin` was fit against the OLD
+ net's (v3) accumulator specifically, before this session's retrain.
+ It happens to be the correct vintage for trial 5 (v3 + v3's own
+ probe), so that comparison is internally consistent, but no probe
+ exists yet for the matprobe net -- `typhoon_nnue`'s wrapper script
+ was updated this session to point at the matprobe net by default
+ and to NOT fall back to the stale probe file (previously silent and
+ wrong). A new probe would need to be fit + calibrated from scratch
+ against the matprobe net's accumulator before it could be used in a
+ trial 4-equivalent run.
+- **A user-remembered "NNUE competitive or better" result could not
+ be reproduced under any of these five conditions.** Likely
+ explanations, unconfirmed: a different match configuration entirely
+ (not `st 4` or `sd 10` against this head binary), or a misremembered
+ result. Do not plan further work assuming that result is real until
+ it can be reproduced under a specific, documented configuration.
+
+### Pipeline sanity check: ruled out a structural bug
+
+Given five straight losses, including at matched depth where ECM
+predicted an edge, the obvious question was "is the training pipeline
+itself broken" (bad feature indexing under real gradient flow, broken
+accumulator math, a training-loop bug) rather than a genuine
+net-quality gap. Tested directly (`annotate_pgns/
+overfit_sanity_check.py`, new tool): sampled 3000 real positions,
+scored each with the engine's own classic `Eval()` (fast
+persistent-context `eval` command), and trained the *same*
+architecture to directly regress raw output against that score (plain
+MSE, no sigmoid, no K, no λ, no quiet filter, no quantization -- the
+simplest possible test of "can this architecture memorize a small,
+clean, noiseless target at all"). **Result: clean overfit** -- RMSE
+collapsed from 307cp to ~3-5cp over 400 epochs on the fixed 3000-point
+set (max error 46cp). **No structural bug found.** The five
+head-to-head losses are real evidence about net/training quality, not
+a pipeline defect.
+
+### Revised understanding of why: likely bootstrapping off HCE's own
+### search sets a ceiling, not (as earlier argued) label noise or
+### architecture capacity
+
+Two explanations floated earlier in this investigation don't hold up
+well under scrutiny:
+- **Architecture capacity**: the 256x2-32-32-1 shape is the literal
+ one Stockfish's own early, successful NNUE work used. Not a good
+ suspect.
+- **"Search-score labels are noisy"**: standard practice (`sd 8`-ish
+ search-score training is exactly what real NNUE projects use); the
+ depth/source of search isn't inherently suspect.
+
+**The sharper version of the concern**: every label in the current
+training pool -- TWIC and self-play both -- comes from THIS SAME
+engine's own classic `Eval()` run through its own `sd 10` search, and
+training used `λ=1.0` (pure `p_search`, zero weight on actual game
+`r_outcome`). A net trained to perfectly reproduce "HCE's static eval,
+distilled to one forward pass" can at best approach HCE's *positional
+judgment* -- it has no training signal that could let it exceed HCE's
+actual playing strength, which is that judgment plus a mature,
+decades-tuned full search. This is single-generation distillation
+from the exact opponent being benchmarked against, with no
+independent ground truth in the loop. Real NNUE bootstrapping success
+stories (including Stockfish's own) iterate through many generations,
+each net's own self-play becoming the next round's data, with game
+outcomes mattering throughout -- not one shot of pure
+distillation-from-one-engine's-shallow-search.
+
+**This reframes the earlier-rejected λ-sweep as an open question
+again, not a settled one** (see the "λ-sweep methodology was weaker
+than later standards" note above) -- a properly-controlled λ=0.5 run
+(combined with the material-probe fix, trained to real convergence,
+judged with the cross-sectional population check, not a single
+hand-picked position) is in progress as of this writing to answer
+this properly.
+
+### Next up
+
+1. **In progress**: λ=0.5 + material-probe-loss combined run, trained
+ to convergence, judged with `sanity_check_net.py --cross-sectional`
+ -- the properly-controlled version of the λ-sweep question above.
+2. Once a net shows a real head-to-head result against HCE (win, draw,
+ or a meaningfully-less-bad loss than trial 4's 0.133), THEN invest
+ in margin recalibration (EFP/LMR/NMP/futility, still 100% HCE-tuned
+ today) and a new lazy-probe fit for whichever net is being kept --
+ not before, per this session's experience chasing search-side
+ explanations that turned out to not be the dominant factor.
+3. Data volume scale-up (TWIC + self-play, more self-play generations
+ specifically using the matprobe-or-later net to play, per the
+ original Phase 2 plan) remains the most clearly-actionable lever,
+ but should be paired with an actual outcome-grounding signal in the
+ training target, not just more HCE-sd10-labeled positions of the
+ same shape already in hand.
+
+## Status (2026-10-18): Stockfish-labeled relabel in progress; feature
+## factorization planned as the next architecture-side lever
+
+Labels are being regenerated using Stockfish (not this engine's own
+`Eval()`/`sd 10`) to break the single-generation self-distillation
+problem described above -- that's a speed/ground-truth fix, not
+believed to be the root cause of the five head-to-head losses by
+itself. In parallel, researching Stockfish's own NNUE docs surfaced a
+second, independent candidate explanation worth trying once there's
+enough Stockfish-labeled data to re-run training: **feature
+factorization** (Stockfish's term for a training-only "virtual
+feature set" that is coalesced into the real weights post-training;
+see https://github.com/official-stockfish/nnue-pytorch/blob/master/docs/nnue.md#feature-factorization,
+excerpted into this conversation).
+
+### Why this looks like a real lever here, not just a ported idea
+
+The current feature set is HalfKAv2_hm per the canonical spec above:
+22528-dim, king-bucketed (32 buckets via `king_bucket()`'s
+horizontal-mirror quartet), so every `(piece_square, piece_type,
+piece_color)` combination exists as up to 32 separate rows, one per
+king bucket, each seen only when the king happens to sit in that
+bucket. Checked directly against the current real training cache
+(`runs/sd10_twic_2M/tensors_train`, 1,848,411 positions):
+
+- **23.9% of the 22528 HalfKAv2 rows have zero observations** --
+ never fire once in training, never receive a gradient update.
+- **60.5% have fewer than 50 observations.**
+- Collapsing the same data to the king-bucket-independent "P" feature
+ set (704 rows: `piece_square x piece_type x piece_color`, no king
+ info) gives a median of **8375** observations per row -- roughly
+ 32x denser, as expected, with none left permanently untrained.
+
+So a substantial fraction of the net's weight space is currently
+stuck at random init or barely trained regardless of label quality --
+plausibly part of why two different training attempts "worked"
+(loss went down, no crash) but neither net could beat HCE in
+head-to-head play. This is Stockfish's own "dusty corners" framing:
+sparsity from king-conditioning starves rare (king, piece, square)
+combinations of signal, which factorization is specifically designed
+to fix during early training.
+
+### Planned implementation (not yet done -- next after enough
+### Stockfish-labeled data exists to re-run training)
+
+1. **No change to `build_training_tensors.py` / no cache rebuild
+ needed.** The HalfKAv2_hm index formula is
+ `idx = (sq ^ mask) + piece_plane + king_bucket * PS_NB` where
+ `(sq ^ mask) + piece_plane <= 703 < PS_NB (704)`. So the P-factor
+ index is exactly `real_idx % 704` -- recoverable from the existing
+ `white_idx`/`black_idx` tensors at train time, in `nnue_model.py`.
+2. **`nnue_model.py`**: add a second embedding table,
+ `self.ft_p = nn.EmbeddingBag(704, l1, mode="sum")`, **zero-initialized**
+ (matters: both tables sum into the same accumulator, so a
+ default-init P table would double the accumulator's scale at step
+ 0, confounding the factorization effect with an effective LR
+ change), with no bias of its own (`ft_bias` stays as the only
+ bias, or coalescing below isn't exact). In `accumulator()`, derive
+ `white_idx % 704` / `black_idx % 704` and sum `ft_p`'s output into
+ each perspective's accumulator alongside `ft`.
+3. **Skip the "K" and "HalfRelativeKP" factors** Stockfish's doc also
+ describes. HalfKAv2_hm already encodes the king directly (kings
+ are included in the `piece_map()` loop via `piece_plane()`'s
+ `PS_KING` case), so the K factor would be redundant here. One
+ variable at a time, per the incremental-change rule used
+ throughout this project -- P alone first, measure, then decide.
+4. **Coalescing after training, in `export_nnue_weights.py`**: this is
+ the step most likely to silently fail, because a bad coalesce
+ produces a net that is *worse* than not factoring at all (the real
+ HalfKAv2_hm rows learn less because the factor absorbed the shared
+ signal) and the training loss curve won't show it either way.
+ `ft.weight += ft_p.weight[torch.arange(22528) % 704]`, then drop
+ both `ft_p` and `material_probe` from the exported weights (two
+ training-only heads now, same pattern). **Required check**: assert
+ the pre-coalesce factored forward pass and the post-coalesce
+ plain-`ft`-only forward pass agree to ~1e-5 on a real batch before
+ trusting the export.
+5. **Evaluation protocol**: Stockfish's doc is explicit that the
+ factorizer's effect is concentrated in *early* training and fades
+ as training progresses -- compare early-epoch loss descent against
+ a matched-seed, matched-epoch unfactored control, not just final
+ validation loss. A final-loss-only comparison could read a real
+ early-training effect as zero.
+
+### Sequencing
+
+This factorization work is queued behind the in-progress Stockfish
+relabel -- the two are independent, complementary levers (ground-truth
+quality vs. training-signal density under sparsity), and per this
+project's incremental-change discipline they should be evaluated one
+at a time rather than conflated in a single before/after comparison.
+
+## Status (2026-10-18, continued): Stockfish-labeled corpus built, P-factor
+## factorization implemented and measured -- why it's load-bearing, not optional
+
+Both levers above are now done, in sequence, against the new
+`pgn/stockfish_self_play_training_data.pgn` corpus (35,295 Stockfish-
+vs-Stockfish games, 4,008,045 positions, labeled via Stockfish's own
+search rather than this engine's `Eval()` -- see `extract_selfplay_
+positions.py`'s `MATE_CLAMP_THRESHOLD=90.0`, needed because this
+foreign-engine corpus encodes mate as a clamped float, `[99.00,99.99]`
+pawns via PolyGlot's UCI-to-xboard bridge, not typhoon's own `+M`/`-M`
+sentinel token). Full pipeline run at `lambda=0.75` (`target =
+0.75*p_search + 0.25*r_outcome`, chosen over the Phase-A `1.0` default
+specifically because pure single-engine-distillation was the leading
+suspect for the prior two nets' failure to beat HCE): `split_holdout.py`
+-> `fit_k_and_label.py` (K=2.4130, this corpus's own fit, NOT the TWIC-
+fit `DEFAULT_K=1.3123` -- the two corpora have different score
+distributions and must not share a K) -> `quiet_filter_games.py` ->
+`build_training_tensors.py` (now with `--workers`, default 16,
+multiprocessing-sharded across positions, verified byte-for-byte
+identical output to the old sequential path -- the per-position
+feature-index computation has no cross-position dependency, so this
+was a clean win on an idle multi-core box).
+
+**Why factorization is necessary here, not a nice-to-have**: measured
+directly on this corpus's real training tensors (not a TWIC estimate),
+2,237 of HalfKAv2_hm's 22,528 rows (9.9%) never fire once across all
+2.84M training positions -- confirmed, after training, to be **bit-
+for-bit identical** to their random pre-training init (diffed the
+trained checkpoint's `ft.weight` against a freshly re-seeded,
+untrained `NnueNet()` at the same seed: max difference on those rows
+is exactly 0.0, vs. a clearly nonzero shift on every observed row).
+This 9.9% figure is *better* than TWIC's 23.9% (this corpus's
+decisive, played-to-conclusion games naturally cover more king
+positions than human-resignation-biased TWIC), but still real, and
+**not fixable by collecting more data at any practical scale**:
+subsampling the existing pool at increasing sizes and fitting the
+shrinkage curve (`zero_count ~= 5529 - 226*ln(N)`, log-linear, fit on
+6 points from 10% to 100% of the current 2.84M-position pool) says
+halving the dead-cell count to ~1,118 needs ~290M positions (~100x
+the current pool), and getting under 500 needs ~4.4B (~1,560x). Brute-
+force self-play volume cannot close this gap; it needs an
+architecture-side fix, which is exactly what factorization is for.
+
+**Implementation** (`nnue_model.py`, `export_nnue_weights.py`), per
+the plan above, built essentially as specified: a second, zero-
+initialized `ft_p` `EmbeddingBag(PS_NB=704, l1)`, summed into the
+accumulator alongside `ft` via `idx % PS_NB` (same bag offsets, no
+`build_training_tensors.py` changes needed), coalesced into `ft.weight`
+at export time (`ft.weight += ft_p.weight[arange(22528) % 704]`) with
+a required pre/post-coalesce forward-pass agreement check (measured
+3.58e-07 on a random-weighted smoke test, comfortably inside the
+~1e-5 tolerance) before `ft_p` is dropped from the exported file.
+
+**Measured effect, matched-seed/epoch (lambda=0.75, K=2.4130,
+material-loss-weight=0.001 -- see the next note on why that changed
+from 0.01, 34 epochs, otherwise identical hyperparameters)**:
+- Holdout MSE at matched checkpoints: factored beats unfactored at
+ every logged step, not just early training (contra the "effect
+ fades" caveat in the plan above, which turned out not to dominate
+ here) -- e.g. step 4000/epoch 11: 0.01193 (factored) vs. 0.01446
+ (unfactored); final epoch 33: 0.01201 (factored) vs. 0.01331
+ (unfactored).
+- `sanity_check_net.py --cross-sectional` (2000 sampled holdout
+ positions, final checkpoints): both nets monotonic in piece value,
+ but factored's queen/pawn ratio is closer to nominal (~8.1x own /
+ 9.8x opponent-removed, vs. the already-decent unfactored run's
+ ~8.2x/9.9x -- both well above the old TWIC-trained matprobe net's
+ ~5.6x).
+- Compared against a REAL Stockfish net's NNUE output (see next
+ section) on 1000 shared holdout positions: factored wins on every
+ metric -- correlation 0.879 vs 0.861, mean|diff| 0.776 vs 0.852
+ pawns raw, 0.718 vs 0.762 pawns after fitting away the overall
+ scale difference (both nets run at ~0.79-0.83x Stockfish's scale,
+ expected since ours is WDL-sigmoid-calibrated, not raw-centipawn-
+ trained).
+
+**`material-loss-weight` lowered from 0.01 to 0.001 for this corpus**,
+before any of the above: the original 0.01 (see "material-blindness
+fixed" status above) was tuned against TWIC's homogeneous, balanced-
+middlegame material distribution, where the raw material-MSE term
+was naturally small and its weighted contribution stayed well below
+the WDL loss's. This corpus has far more material-imbalanced
+positions by construction (decisive self-play games played to real
+conclusions, not human-resignation-biased), so the same 0.01 weight
+measurably put the material term at parity with the WDL loss rather
+than a gentle nudge -- confirmed by briefly launching training at
+0.01 before catching it (checkpoints `ckpt_epoch000-001.pt` from that
+run were deleted, not used for anything above).
+
+**Known residual gap, not yet fixed**: `ft_p`'s zero-init was
+deliberate (avoids confounding the factorization effect with an
+init-time accumulator-scale change), but `ft` itself still uses
+`nn.EmbeddingBag`'s default `N(0,1)`-per-entry init, which is NOT
+zero -- so an untouched row's random contribution is the same
+magnitude (`norm~16`) as a fully-trained row's, not suppressed by
+scale. Today this is masked by `ft_p`'s well-trained, shared
+correction dominating the sum at those positions (confirmed: 0/1000
+Stockfish-comparison positions activated any dead cell, so this gap
+hasn't tainted any reported result above), but it's a real residual
+risk for any future position that activates several dead cells
+together. Cheap, not-yet-applied fix: zero-init `ft` the same way
+`ft_p` already is, so a dead cell degrades gracefully to "contributes
+nothing" rather than "contributes untrained noise."
+
+## Verifying against a real Stockfish net (one-time diagnostic, not a
+## pipeline dependency)
+
+To sanity-check whether the above was in the right ballpark at all (as
+distinct from reusing Stockfish's own weights, which was explicitly
+ruled out -- the goal is a from-scratch net strong enough to self-play
+and iterate on, "eating its own tail," not a transplant), pulled a
+real, period-appropriate Stockfish net (`nn-fdc1d0fe6455.nnue`, added
+to Stockfish 2023-06-12 per `github.com/official-stockfish/networks`'
+commit history, rev `38e61663d836e062af0bc002814ad5149c4b7729`) and
+parsed its `FeatureTransformer::psqtWeights` table directly, against
+that exact commit's header-reading code (fetched and read, not
+guessed -- a first attempt at parsing by eye got the header layout
+wrong and was caught before trusting any conclusion from it).
+
+- **Confirmed identical feature space**: `HalfKAv2_hm::HashValue =
+ 0x7f234cb8` and `Dimensions=22528` match our own spec exactly (same
+ king buckets, same orientation table, same piece-square planes);
+ this net's stored `ft` hash decodes to `L1=1536` (vs. our 256) via
+ `ft_hash = HashValue XOR (2*L1)`.
+- **`psqtWeights[index][bucket]` for our own 2,237 dead-cell indices
+ looks completely ordinary** relative to that plane's overall average
+ (e.g. queen plane: 2487cp mean on dead cells vs. 2541cp overall) --
+ i.e. our dusty corners aren't inherently weird positions, just a
+ sampling gap in this specific corpus, which is exactly the pattern
+ factorization is built to exploit (the true per-piece value is
+ roughly king-bucket-independent).
+- To get a real NNUE eval from this old net for the comparison above,
+ built Stockfish from source at the exact matching commit
+ (`gmake -j$(nproc) build ARCH=x86-64-avx2 COMP=clang` -- note
+ FreeBSD's default `make` is BSD make and cannot parse this
+ Makefile's GNU-Make syntax, must use `gmake`), rather than risk
+ hand-reimplementing its 8-layer-stack quantized forward pass.
+ Compared its `eval` command's "NNUE evaluation" line (pure NNUE
+ output, no classical-blend) against our own `forward()`, both
+ converted to white-relative, over 1000 shared holdout FENs -- see
+ `annotate_pgns/compare_vs_stockfish_net.py` and the factorization
+ status note above for the numeric results.
+
+## Iterating on a net: adding a new generation of self-play data
+
+The first net to actually beat (or seriously contest) HCE at matched
+`sd 10` is the `lambda=0.75`, P-factored checkpoint from the status
+above -- the point where self-play with this net starts producing a
+genuinely bigger/more-targeted training pool becomes real ("eating its
+own tail", per the conversation that drove this project's current
+phase). This section is the checklist for folding a new batch of
+self-play games into the next training generation -- written down so
+it doesn't need re-deriving from scratch (or re-making the same
+mistakes) each time. **None of this requires a new architecture or a
+new `nnue_model.py` -- same model class, same feature set, same
+`--resume` mechanism.** It's a data/calibration discipline problem,
+not a code problem.
+
+### 1. Combine, don't just append
+
+Pool the new self-play positions together with the existing training
+corpus (corpora can mix freely at the games.jsonl level -- see
+`extract_selfplay_positions.py`'s docstring) and re-run the **full**
+pipeline on the combined set:
+
+```
+split_holdout.py -> fit_k_and_label.py -> quiet_filter_games.py -> build_training_tensors.py
+```
+
+Do NOT train on only the new self-play batch by itself, even via
+`--resume`. Self-play games cluster around positions the current net
+already favors (that's what self-play means) -- training on only the
+new batch narrows the effective training distribution and risks
+forgetting what the original, more diverse corpus taught. The combined
+pool is also exactly how the dusty-corner coverage problem (see the
+factorization status above) actually improves over time: new self-play
+games sample different king positions/endgame structures than the
+corpus that came before, so the pooled coverage count should be
+re-measured after each new generation (same `np.bincount` check used
+throughout this doc) to see whether it's actually shrinking the dead-
+cell count, not just assumed.
+
+### 2. Refit K on the combined pool -- never carry an old K forward
+
+`fit_k_and_label.py`'s K is fit to one specific corpus's own score
+distribution. **"Fitted K = 2.4130 -- record this; applying a different
+K later (e.g. after labeling more data) is not the same model"** (the
+tool's own exit message) applies in the other direction too: once the
+pool changes, K must be refit on the new combined TRAIN split, not
+reused from the previous generation. Training against a target
+calibrated for a different score distribution than the one actually
+being trained on silently miscalibrates every label, the same class of
+problem as DNA-tuning's holdout-leakage concerns elsewhere in this
+project.
+
+### 3. Watch the label source -- this is the one that actually matters
+
+How were the new self-play games scored?
+
+- **If an independent, stronger reference (Stockfish, or any engine
+ not being trained) produced the centipawn labels**, this is safe
+ regardless of which engine(s) played the moves -- self-play between
+ two candidates is a great way to source diverse, targeted positions
+ (specifically ones that stress whatever the current net is weakest
+ at), as long as the position doesn't then get "approved" by asking
+ the same net being trained to grade itself.
+- **If the candidate net's own search produced the score labels for
+ its own games**, this reintroduces the single-generation self-
+ distillation problem already diagnosed as the likely reason the
+ first two (pre-Stockfish-relabel) nets failed to beat HCE at all
+ (see the "Revised understanding of why" status note above) -- a net
+ trained to reproduce its own judgment can at best match that
+ judgment, never exceed it. Mitigate, don't ignore, by keeping
+ `lambda < 1.0` in `build_training_tensors.py` so real game outcomes
+ (`r_outcome`) stay in the training target regardless of where the
+ centipawn half of the blend came from -- outcomes are ground truth
+ no self-distillation loop can corrupt, even when the score labels
+ themselves are suspect.
+
+### 4. Re-check `material-loss-weight` against the new pool's material
+ distribution
+
+`material-loss-weight` was lowered from `0.01` (tuned for TWIC's
+balanced-middlegame distribution) to `0.001` specifically because this
+project's Stockfish self-play corpus has far more decisive, material-
+imbalanced games. A new self-play generation's material distribution
+is a new empirical fact, not something to assume stays `0.001` forever
+-- if candidate-vs-candidate games turn out far more drawish/balanced
+than the original bootstrap corpus (plausible once both sides are
+playing well), the same weight could again land at an unintended
+fraction of the total loss. Check, don't assume -- same reasoning as
+the original 0.01->0.001 change, just potentially in either direction
+next time.
+
+### 5. Warm-start, don't restart
+
+`--resume <previous checkpoint>` on the new combined-pool tensors is
+the right default -- it's strictly faster than random-init-and-train-
+from-scratch, and there's no evidence in this project so far that
+warm-starting hurts final quality (nothing here has tested the
+opposite, to be clear -- if a future generation's holdout metrics look
+worse than a from-scratch control at the same epoch count, that's
+worth isolating before trusting warm-starting blindly forever).
+
+### 6. Validate before trusting the new generation
+
+In order, cheapest/fastest first -- stop at the first one that fails
+rather than running all of them blind:
+
+1. `sanity_check_net.py --cross-sectional` on the new checkpoint --
+ monotonic piece-value deltas, reasonable magnitudes (compare
+ against the queen/pawn ratio of the previous generation as a
+ baseline, not just "is it monotonic").
+2. Re-run the dead-cell coverage check (bincount over the new combined
+ training tensors) -- confirm it's trending down, not flat or up.
+3. `compare_vs_stockfish_net.py` against the same real Stockfish net
+ used above, same held-out FEN sample where practical, to see
+ whether correlation/scale-fit-residual improved or regressed vs.
+ the previous generation's numbers (0.879 corr / 0.718 pawns resid.
+ as of the factored lambda=0.75 checkpoint -- the number to beat).
+4. `export_nnue_weights.py` (coalesce check must pass, zero clipped
+ values) then a real matched-depth (`sd 10`, not `st` -- see the
+ fixed-depth-for-ECM-comparisons convention used throughout this
+ project) head-to-head via `eval_tune/test_vs_head.sh` with
+ `NNUE_WEIGHTS` set, against classic HCE. This is the only step that
+ actually answers "did this generation get better," everything above
+ is a cheaper proxy that can catch a regression before burning a
+ multi-hour match run on it.
+
+## Adoption criterion: sd-equal parity is not the bar, `st`-equal is --
+## and the net is currently ~3.3x slower per node than HCE
+
+Measured directly in the first real sd10-matched match (the factored
+`lambda=0.75` checkpoint vs. classic HCE, 2026-10-18, 400 games):
+candidate nps=470-475K vs. head nps=1.5-1.6M -- **a ~3.3x gap**, not a
+rounding error. At matched *depth* this is invisible (both sides reach
+sd10 regardless of nps), which is exactly why the sd-equal match is
+the right first filter -- it isolates "is the net smarter per node"
+from the speed handicap. But a `st`-equal (real time control) match
+would currently be lopsided in HCE's favor for a reason that has
+nothing to do with eval quality: at this project's branching factor
+(ebf ~3.8-3.9, measured in the same match), a 3.3x nps disadvantage
+costs roughly a full ply of search depth at any fixed time budget, on
+top of whatever the eval-quality gap already costs. Testing at `st`
+before closing that gap would answer "how bad is an engine with no
+lazy-eval fast path," not "is this net any good."
+
+**Revised understanding (2026-10-18, same conversation): nps parity is
+sufficient, not a blocker -- the real Stockfish binary on this exact
+box is the proof.** It searches at ~500K nps -- essentially the same
+as our net's measured 470-480K, not the 1.5-1.6M HCE achieves -- and
+still beats this project's HCE every single recorded game (not even a
+draw). Stockfish doesn't out-node HCE; it out-*prunes* it, searching
+dramatically trimmer trees at the same or worse raw nps, because its
+eval is trustworthy enough to prune aggressively on. This reframes the
+priority between the two levers below: **lazy eval (closing the 3.3x
+nps gap) is a nice-to-have, not the thing standing between this net
+and real strength.** The actual lever is section 3 of the checklist
+below -- null-move R, LMR's reduction table and fail-high prediction,
+LMP thresholds, futility margins -- all of which currently encode how
+*little* HCE's eval can be trusted. A learned eval earning the right to
+prune harder, the way Stockfish's does, is plausibly worth more than
+recovering the nps gap ever could be. Lazy eval stays worth doing (free,
+no downside), just not gated-on-first the way the staged list below
+originally implied.
+
+**Staged gate, in order -- do not skip ahead:**
+
+1. **Sd-equal match (done, 2026-10-18)**: necessary-but-not-sufficient
+ filter. If the net can't hold its own at matched depth, it has no
+ chance at matched time, since real time controls only make the
+ speed handicap worse for it, never better.
+2. **Recalibrate the pruning-aggressiveness constants (checklist
+ section 3) -- this is the primary lever, per the Stockfish
+ comparison above, not a secondary cleanup pass.** Not a full
+ DNA-tuning sweep, just enough that search isn't making prune/
+ reduce/extend decisions calibrated for a different eval's scale and
+ trustworthiness. Plausibly worth more than nps recovery on its own.
+3. **Close the free speed gap too** -- lazy eval (checklist section 1),
+ still worth doing since it's pure engineering with no downside (it
+ doesn't change what the net *thinks*, only how often it's asked to
+ think at nodes that get pruned anyway) -- just not the bottleneck
+ the Stockfish comparison suggested it might be.
+4. **Only then, a real `st`-equal SPRT against HCE.** This is the step
+ that actually answers the adoption question -- everything before it
+ answers a necessary precondition, not the question itself.
+
+**The adoption threshold itself shouldn't be pure parity-or-better,
+though.** HCE is a mature, hand-tuned eval already deep into
+diminishing returns on its own DNA-tuning pipeline. NNUE has an
+open-ended improvement path still mostly unexploited (more self-play
+generations per the iteration section above, factorization already
+landed, a wider L1 if a Phase B GPU run ever happens -- see the real
+Stockfish net's L1=1536 vs. our 256 as a sense of the room available).
+A net that's close-but-not-quite-parity at a genuinely fair (`st`-
+equal, post-calibration) time control may still be worth adopting if
+the plan is to keep training it -- the alternative is maintaining two
+eval paths indefinitely while one's ceiling keeps rising and the
+other's doesn't. Close-and-improving beats even-and-stagnant.
+
+## Engine calibration checklist: what needs re-tuning for a new eval
+
+This engine's search has several subsystems whose margins/tables were
+fit against classic `Eval()`'s specific output scale and noise
+profile. None of this is required for the net to *run* (it already
+does, correctly, per the wiring in `search.c`/`nnueeval.c`) -- it's
+required for the net to run *well*, since an uncalibrated search is
+fighting the new eval with pruning/reduction decisions tuned for a
+different one. Confirmed against this engine's actual current search
+(2026-10-18 conversation; EFP and ProbCut were previously assumed
+present and are not -- corrected below rather than left as dead
+checklist items):
+
+**1. Free speed, no behavior change (do this first, before any of the
+below -- see the adoption-criterion note above for why ordering
+matters here):**
+- Lazy eval fast path -- designed (architecture decisions above: reuse
+ the net's own cheap linear/PSQT-ish term -- here `ft_p`/
+ `material_probe`-shaped -- as a first-pass filter, falling through
+ to the full forward pass only when inconclusive vs. alpha/beta), not
+ yet built. This is the single biggest lever on the measured 3.3x nps
+ gap and touches zero tuning -- it's the right first move specifically
+ because it can't make the net play worse, only faster.
+- Deferred/lazy accumulator maintenance (flagged in the accumulator
+ design section above as a "performance follow-up, not required for
+ correctness") -- avoid eager accumulator updates at nodes pruned
+ before any eval is ever consulted.
+
+**2. The raw-score-to-cp mapping -- verify this before touching any
+margin below, since it's a multiplier on all of them:**
+Every margin in section 3 is a flat cp (or near-cp) threshold tuned
+against HCE's output scale. The trained net runs at ~0.8x a real
+Stockfish net's scale (measured via `compare_vs_stockfish_net.py`) and
+is WDL-sigmoid-calibrated (K=2.4), not raw-centipawn-trained. If
+search currently treats `forward()`'s raw output as if it were
+already in HCE's cp convention, every margin below is off by that
+scale factor before even considering noise-profile differences. This
+needs the inverse-sigmoid-to-cp conversion the original architecture
+decision already called for ("Requires a calibrated inverse-sigmoid
+conversion back to centipawns... that conversion constant needs its
+own calibration pass") -- confirm it exists and is right once, rather
+than chasing a scale error through a dozen separately-miscalibrated
+margins.
+
+**3. Score-scale/noise-dependent pruning and reduction constants --
+this engine's actual current set (corrected 2026-10-18: no EFP in main
+search, only LMP; no separate razoring step -- LMR fills that role
+here; no ProbCut; QSearch futility and "delta pruning" are the same
+mechanism, not two):**
+- QSearch futility margins
+- Static/reverse null-move pruning margin
+- Null-move reduction depth R, and any eval-based skip/verification
+ gating on the null-move try itself
+- LMP move-count table, `data.c`
+- LMR reduction table, `data.c`, **and** the joint fail-high
+ prediction table feeding it (see the LMR sd11/sn10M protocol note
+ elsewhere in this project) -- this table is a direct statement about
+ how trustworthy the eval is at a reduced-depth node, fit entirely
+ against HCE's noise profile
+- Root aspiration window initial width and re-widening schedule on
+ fail-high/fail-low (tuned against HCE's iteration-to-iteration score
+ noise -- a good catch, easy to overlook since it's at the root, not
+ buried in the main search loop)
+
+**4. Eval-component-dependent heuristics (lower priority -- already
+confirmed to degrade gracefully under a lazy/missing full eval per the
+"Breaking external dependencies" section above; re-measure how often
+they actually fire once NNUE lazy eval ships, don't re-derive from
+scratch):**
+- `EstimateHangingMaterialCp`'s `cTrapped`-fallback "dumb move" axis
+- `ValueOfMaterialInTroubleDespiteMove`'s QSearch stand-pat gate
+
+**5. Currently dormant, no action needed yet:** draw/contempt scoring
+is built but always returns 0 right now -- no live miscalibration risk
+today, but if it's ever turned on, the same cp-scale concern in
+section 2 applies to whatever constant drives it.
-## Immediately actionable, unblocked work
+## Status (2026-10-18, continued): pipeline-scaling audit, three gaps fixed, end-to-end driver added
-Not gated behind any engine-internals decision above:
+Prompted by generating a much larger volume of annotated PGN than
+before and wanting to confirm the post-annotation Python pipeline
+(`annotate_pgns/`) actually uses all available parallelism rather than
+silently bottlenecking on a single-threaded stage. `annotate.py`
+itself (the sd-N search stage) was already confirmed fine -- sharded
+across N single-threaded engine subprocesses -- and
+`build_training_tensors.py` already had a `--workers 16`
+`multiprocessing.Pool`. Three real gaps found and fixed; one
+unrelated question (an end-to-end PGN-to-`.nnue` driver) answered by
+adding a new orchestrator script.
-- Build the annotation harness and begin scoring `lichess_elite_*`/
- `twic_filtered.pgn` games (Phase 1 training data).
-- Investigate `<TT>`/`<EGTB>`/`<REC>` tag semantics in the self-play PV
- printer.
-- Finalize the written HalfKAv2_hm feature-indexing spec (design-only,
- no code).
+- **Gap: `quiet_filter.py` had no parallelism at all.** It's a
+ single-threaded Python loop doing `chess.Board()` construction + SEE
+ per row over the entire corpus (2.5M+ rows for the Stage A run) --
+ the one CPU-bound step in the non-annotation part of the pipeline
+ with zero parallelism, unlike `build_training_tensors.py`'s `Pool`
+ right next to it. **Fixed via a new wrapper, `shard_quiet_filter.py`**,
+ rather than rewriting `quiet_filter.py` itself: round-robin-splits
+ the input across `--workers` (default 16) temp shard files, runs
+ that many `quiet_filter.py` subprocesses concurrently, concatenates
+ their outputs. Deliberately reuses `quiet_filter.py` unmodified --
+ one definition of the filter, reused as a subprocess, the wrapper
+ only owns sharding/recombining. Row order in the combined output
+ does not match the input (shards are concatenated shard-by-shard,
+ not interleaved back to original order) -- harmless here since
+ `positions.quiet.jsonl` is a flat, order-independent training-set
+ input. Verified end-to-end on a synthetic 40-row fixture: 40 in, 40
+ out, across 4 shards.
+- **Gap: `fit_k_and_label.py`'s ternary-search K-fit was unvectorized
+ Python.** `fit_k()`'s 40 ternary-search iterations each call `mse()`
+ twice, and the old `mse()` was a plain Python `for` loop over every
+ (score, outcome) pair -- roughly 80 full passes over ~2M+ pairs in
+ pure Python for a step that does very little actual compute.
+ **Rewritten to numpy** (`sigmoid`/`mse` as vectorized array ops over
+ the full pair set) **and sharded** across `--workers` (default 16)
+ via `multiprocessing.Pool`, for *both* halves of the file: the
+ calibration-pair collection (`collect_calibration_pairs`, now
+ `collect_chunk` workers each vectorizing one chunk of games with
+ numpy, combined via `np.concatenate`) and the labeling pass
+ (`label_file`, now `label_chunk` workers, each vectorizing one
+ game's non-exact positions' sigmoid computation per chunk) --
+ `Pool.imap`-based, order-preserving, mirroring
+ `build_training_tensors.py`'s existing chunking pattern. Verified
+ end-to-end on a synthetic 50-game fixture (4 workers, chunk-size 7):
+ correct game/position counts, K fit, both splits labeled.
+- **Gap: `nnue_train.py`'s CPU thread default was lower than the rest
+ of the pipeline's convention.** `--threads` default bumped 8 -> 16,
+ matching the `--workers 16` convention used everywhere else in this
+ pipeline (kept below this box's 24 cores -- see below -- rather than
+ maxed out, so other jobs on the shared box keep some breathing
+ room).
+- **Core-count note, not acted on:** this box has 24 cores
+ (`nproc`), so every one of these `--workers`/`--threads`/
+ `annotate.py --shards` defaults (16) intentionally leaves some
+ headroom rather than using all 24 -- a deliberate choice to leave
+ the box's other jobs breathing room, not an oversight. `annotate.py`
+ Stage A's own measured throughput (NNUE.md's "Status (2026-10-14)"
+ section: ~75 pos/sec aggregate at 16 shards vs. ~112/sec
+ extrapolated from one shard x16, attributed to contention) is a
+ reminder that this box's parallelism doesn't scale linearly past a
+ point anyway -- blindly raising shard/worker counts further is not
+ assumed to help without measuring first.
+- **New: `run_nnue_pipeline.py`**, an end-to-end orchestrator from a
+ glob of input PGN to a `.nnue` weights file, requested directly as
+ "can one script run the whole training process end to end." Targets
+ **already-scored PGN** specifically (match_play.py's
+ `"{+-}NN.NN/DEPTH PV..."` per-move comment format -- e.g.
+ `pgn/stockfish_self_play_training_data.pgn`, `pgn/self_play_games.pgn`),
+ not raw human-game PGN needing a fresh sd-N search -- that's
+ `annotate.py`'s job and is a different, not-yet-automated entry
+ point. Chains, in order: `extract_selfplay_positions.py` (once per
+ input file, since that script takes a single PGN; outputs
+ concatenated) -> `split_holdout.py` -> `fit_k_and_label.py` ->
+ `quiet_filter_games.py` (both splits) -> `build_training_tensors.py`
+ (both splits) -> `nnue_train.py` -> `export_nnue_weights.py` (against
+ the latest `ckpt_epochNNN.pt` written -- `nnue_train.py` has no
+ separate "best checkpoint" selection yet, so "latest" is the only
+ option today). Every stage checks whether its expected output
+ already exists and skips it unless `--force` is passed -- the later
+ stages here can run for hours, so a crash/interrupt partway through
+ should not mean restarting from stage 1. Passes `--workers`/
+ `--threads` through at the same default of 16 used everywhere else.
+ A thin orchestrator by design: shells out to the existing,
+ individually-tested scripts via `subprocess` rather than
+ reimplementing any of their logic. Argument parsing, `--help`, and
+ the glob-miss error path were smoke-tested; the full multi-stage run
+ (which needs the engine binary, real PGNs, and hours of wall clock)
+ was not run end-to-end as part of this change.
-Gated behind opening a dedicated branch (per discussion -- not yet
-started):
+## TODO (not yet implemented)
-- Any `SEARCHER_THREAD_CONTEXT`/`PLY_INFO`/MakeMove/UnmakeMove changes.
+- **Horizontal-flip data augmentation, in the position loader
+ (`build_training_tensors.py`).** HalfKAv2_hm's king-bucket mirroring
+ does NOT tie piece-square weights across the a/h files -- the
+ `OrientTBL` XOR step re-orients the board onto the king's own half
+ but a piece's *file within that half* still maps to a distinct
+ feature row, so a↔h-mirrored positions are not already folded
+ together by the architecture. Randomly mirroring positions
+ left-right at load time (flip every square's file, recompute both
+ perspectives' indices, swap castling rights symmetrically where
+ applicable) is a legitimate, standard way to roughly double the
+ effective sample count per feature row without changing dataset
+ size on disk. Directly relevant given the measured sparsity
+ (23.9% of HalfKAv2 rows with zero observations, per the
+ 2026-10-18 factorization status above) -- should be evaluated
+ alongside P-factorization, not instead of it, since the two
+ address the same sparsity symptom through different mechanisms
+ (more effective samples per row vs. a denser auxiliary feature
+ set) and their effects may not be additive.
+- **L2 weight decay in `nnue_train.py`'s optimizer (AdamW).** Pulls
+ rarely-updated weights (the same sparse/underexposed rows the
+ factorization work above targets) toward zero instead of leaving
+ them to drift on whatever noisy gradient they happened to last
+ receive. Cheap -- a single optimizer hyperparameter, no
+ architecture or export-format change -- so there's no reason to
+ gate it behind the factorization/augmentation experiments above;
+ can be added and swept independently.
diff --git a/src/eval.c b/src/eval.c
index 76b332c..de59dd3 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -3275,6 +3275,12 @@ Return value:
static UINT64 g_u64SuperLazyEvalsByBucket[BASE_MARGIN_MAT_BUCKETS];
static UINT64 g_u64LazyEvalsByBucket[BASE_MARGIN_MAT_BUCKETS];
static UINT64 g_u64FullEvalsByBucket[BASE_MARGIN_MAT_BUCKETS];
+// Same idea, for the two NNUE exits -- added 2026-10-16 alongside the
+// EVAL_POSITIONAL_SOURCE_NNUE_FULL/NNUE_LAZY tiers, specifically to
+// answer "what fraction of real NNUE eval calls does the lazy probe
+// gate actually resolve" with a real measurement instead of a guess.
+static UINT64 g_u64NnueLazyEvalsByBucket[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_u64NnueFullEvalsByBucket[BASE_MARGIN_MAT_BUCKETS];
void
DumpEvalExitRateByBucket(void)
@@ -3324,6 +3330,29 @@ Return value:
100.0 * (double)u64L / (double)u64Total,
100.0 * (double)u64F / (double)u64Total);
}
+
+ Trace("NNUE eval exit rate by material bucket (lazy probe gate vs full "
+ "forward pass):\n");
+ for (m = 0; m < BASE_MARGIN_MAT_BUCKETS; m++)
+ {
+ UINT64 u64NL = g_u64NnueLazyEvalsByBucket[m];
+ UINT64 u64NF = g_u64NnueFullEvalsByBucket[m];
+ UINT64 u64Total = u64NL + u64NF;
+ if (0 == u64Total)
+ {
+ Trace(" bucket %u (scaler %u-%u): no samples\n",
+ m, m * BASE_MARGIN_MAT_WIDTH,
+ (m * BASE_MARGIN_MAT_WIDTH) + BASE_MARGIN_MAT_WIDTH - 1);
+ continue;
+ }
+ Trace(" bucket %u (scaler %2u-%2u): n=%12" COMPILER_LONGLONG_UNSIGNED_FORMAT
+ " nnue_lazy=%5.2f%% nnue_full=%5.2f%%\n",
+ m, m * BASE_MARGIN_MAT_WIDTH,
+ (m * BASE_MARGIN_MAT_WIDTH) + BASE_MARGIN_MAT_WIDTH - 1,
+ u64Total,
+ 100.0 * (double)u64NL / (double)u64Total,
+ 100.0 * (double)u64NF / (double)u64Total);
+ }
}
@@ -3494,6 +3523,37 @@ static UINT64 g_uSuperLazyMarginSwingSumSq[BASE_MARGIN_MAT_BUCKETS];
static UINT64 g_uSuperLazyMarginExceeded[BASE_MARGIN_MAT_BUCKETS];
static UINT64 g_uSuperLazyMarginExceededFenSamples[BASE_MARGIN_MAT_BUCKETS];
+// Same again, for the experimental NNUE lazy probe (docs/NNUE.md's
+// "Lazy-evaluation tiering plan") -- |NnueForward() - NnueProbeScore()|
+// swing, same bucketing (_MaterialBucket), same harness, own counter
+// set so this doesn't mix with either HCE tier's numbers. No real
+// margin exists for this tier yet -- this is purely measurement, not
+// gating anything -- so RecordNnueLazyMarginSafetySwing is always
+// called with an effectively-infinite iActualMarginUsed (see its call
+// site in Eval()) until a real margin is derived from this data.
+static UINT64 g_uNnueLazyMarginSwingMax[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueLazyMarginSwingCount[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueLazyMarginSwingSum[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueLazyMarginSwingHist[BASE_MARGIN_MAT_BUCKETS][SWING_HIST_BINS];
+static UINT64 g_uNnueLazyMarginSwingSumSq[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueLazyMarginExceeded[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueLazyMarginExceededFenSamples[BASE_MARGIN_MAT_BUCKETS];
+
+// Material-only "super lazy" tier -- |NnueForward() -
+// pos->iMaterialBalance[uToMove]|, no fit/regression at all (unlike
+// the accumulator probe above): VALUE_PAWN=100 means iMaterialBalance
+// is already in centipawns, the same units NnueForward returns, so
+// raw material balance is directly usable as the "center" estimate --
+// exactly how classic Eval()'s own super-lazy exit already uses it.
+// Same bucketing/harness as the lazy tier above.
+static UINT64 g_uNnueSuperLazyMarginSwingMax[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueSuperLazyMarginSwingCount[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueSuperLazyMarginSwingSum[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueSuperLazyMarginSwingHist[BASE_MARGIN_MAT_BUCKETS][SWING_HIST_BINS];
+static UINT64 g_uNnueSuperLazyMarginSwingSumSq[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueSuperLazyMarginExceeded[BASE_MARGIN_MAT_BUCKETS];
+static UINT64 g_uNnueSuperLazyMarginExceededFenSamples[BASE_MARGIN_MAT_BUCKETS];
+
static void
_RecordMarginSafetySwingInto(IN OUT UINT64 *puMax,
IN OUT UINT64 *puCount,
@@ -3616,6 +3676,84 @@ Return value:
}
+static void
+RecordNnueLazyMarginSafetySwing(IN POSITION *pos,
+ IN SCORE iSwing,
+ IN SCORE iActualMarginUsed)
+/**
+
+Routine description:
+
+ Same shape as RecordSuperLazyMarginSafetySwing, but for the
+ experimental NNUE accumulator-based lazy probe (docs/NNUE.md's
+ "Lazy-evaluation tiering plan") -- |NnueForward() -
+ NnueProbeScore()|, own counter set. No margin is actually gated on
+ yet -- iActualMarginUsed is passed as an effectively-infinite
+ sentinel from Eval() until a real margin is derived from this
+ measurement, so puExceeded stays meaningless for now; only the
+ histogram/sum/sumSq/max/count fields are real data at this stage.
+
+Parameters:
+
+ POSITION *pos
+ SCORE iSwing : >= 0
+ SCORE iActualMarginUsed : sentinel (no real margin chosen yet)
+
+Return value:
+
+ void
+
+**/
+{
+ _RecordMarginSafetySwingInto(g_uNnueLazyMarginSwingMax,
+ g_uNnueLazyMarginSwingCount,
+ g_uNnueLazyMarginSwingSum,
+ g_uNnueLazyMarginSwingSumSq,
+ g_uNnueLazyMarginSwingHist,
+ g_uNnueLazyMarginExceeded,
+ g_uNnueLazyMarginExceededFenSamples,
+ "nnue_lazy",
+ pos, iSwing, iActualMarginUsed);
+}
+
+
+static void
+RecordNnueSuperLazyMarginSafetySwing(IN POSITION *pos,
+ IN SCORE iSwing,
+ IN SCORE iActualMarginUsed)
+/**
+
+Routine description:
+
+ Same shape as RecordNnueLazyMarginSafetySwing, but for the
+ material-only "super lazy" signal (|NnueForward() -
+ pos->iMaterialBalance[uToMove]|) -- own counter set, same
+ sentinel-until-a-real-margin-exists convention.
+
+Parameters:
+
+ POSITION *pos
+ SCORE iSwing : >= 0
+ SCORE iActualMarginUsed : sentinel (no real margin chosen yet)
+
+Return value:
+
+ void
+
+**/
+{
+ _RecordMarginSafetySwingInto(g_uNnueSuperLazyMarginSwingMax,
+ g_uNnueSuperLazyMarginSwingCount,
+ g_uNnueSuperLazyMarginSwingSum,
+ g_uNnueSuperLazyMarginSwingSumSq,
+ g_uNnueSuperLazyMarginSwingHist,
+ g_uNnueSuperLazyMarginExceeded,
+ g_uNnueSuperLazyMarginExceededFenSamples,
+ "nnue_superlazy",
+ pos, iSwing, iActualMarginUsed);
+}
+
+
// Plain Newton's-method sqrt, no <math.h> -- chess.h's own VALUE_KING
// #defines INFINITY_SCORE as a board-eval sentinel, which collides with
// <math.h>'s own INFINITY_SCORE macro, so libm's sqrt() isn't available
@@ -3729,6 +3867,20 @@ Return value:
g_uSuperLazyMarginSwingSumSq,
g_uSuperLazyMarginSwingMax,
g_uSuperLazyMarginSwingHist);
+ _PersistMarginSwingHistogramToFile(f, "nnue_lazy",
+ g_uNnueLazyMarginSwingCount,
+ g_uNnueLazyMarginExceeded,
+ g_uNnueLazyMarginSwingSum,
+ g_uNnueLazyMarginSwingSumSq,
+ g_uNnueLazyMarginSwingMax,
+ g_uNnueLazyMarginSwingHist);
+ _PersistMarginSwingHistogramToFile(f, "nnue_superlazy",
+ g_uNnueSuperLazyMarginSwingCount,
+ g_uNnueSuperLazyMarginExceeded,
+ g_uNnueSuperLazyMarginSwingSum,
+ g_uNnueSuperLazyMarginSwingSumSq,
+ g_uNnueSuperLazyMarginSwingMax,
+ g_uNnueSuperLazyMarginSwingHist);
fclose(f);
Trace("Raw margin-safety histograms written to %s\n", szPath);
}
@@ -3828,6 +3980,18 @@ Return value:
g_uSuperLazyMarginSwingSum,
g_uSuperLazyMarginSwingSumSq,
g_uSuperLazyMarginExceeded);
+ _DumpMarginSafetyCalibrationInto("NNUE lazy probe margin safety",
+ g_uNnueLazyMarginSwingMax,
+ g_uNnueLazyMarginSwingCount,
+ g_uNnueLazyMarginSwingSum,
+ g_uNnueLazyMarginSwingSumSq,
+ g_uNnueLazyMarginExceeded);
+ _DumpMarginSafetyCalibrationInto("NNUE super lazy (material-only) margin safety",
+ g_uNnueSuperLazyMarginSwingMax,
+ g_uNnueSuperLazyMarginSwingCount,
+ g_uNnueSuperLazyMarginSwingSum,
+ g_uNnueSuperLazyMarginSwingSumSq,
+ g_uNnueSuperLazyMarginExceeded);
PersistMarginSafetyHistogramsToFile(szHistogramPathOverride);
}
#endif // CALIBRATE_MARGIN_SAFETY
@@ -4104,7 +4268,7 @@ static const SCORE iKingP98ByAttackerArmy[8] =
// p98/p2 pessimistic/optimistic pairing margin_a/margin_b use --
// p50 is the right statistic for a bias correction, p98/p2 is the
// right statistic for a soundness bound. iPositional[color] =
-// iKingP50ByAttackerArmyOnly[opponent's army bucket] (opponent is the
+// iKingP50ByAttackerArmy[opponent's army bucket] (opponent is the
// attacker of this side's own king) + iResidualP50ByOwnArmy[this
// side's own army bucket] (or the super-lazy-scoped sibling, for the
// super-lazy exit).
@@ -7043,22 +7207,18 @@ Return value:
// intentionally the least-specific answer that's still safe, not
// an attempt to approximate what a real positional estimate would
// say.
- if ((TRUE == g_fUseNnueEval) && (TRUE == g_fNnueWeightsLoaded))
- {
- SCORE iNnueScore = NnueForward(ctx);
- if (NULL != piPositional)
- {
- (*piPositional)[WHITE] = 0;
- (*piPositional)[BLACK] = 0;
- }
- return iNnueScore;
- }
-
- // Initialize army scalers here (moved ahead of the super-lazy exit
- // point below, 2026-09-08 -- their only inputs, uNonPawnMaterial,
- // are already available this early, and the super-lazy margin
- // table needs a material bucket before it can run). We skip lazy
- // eval if we're too late into an endgame.
+ // Initialize army scalers here -- moved ahead of BOTH the NNUE
+ // short-circuit above and the super-lazy exit point below,
+ // 2026-10-15 (was already ahead of super-lazy only, since
+ // 2026-09-08; moved further to also cover the NNUE path). Their
+ // only inputs, uNonPawnMaterial, are already available this early.
+ // Side effect of this move: pos->uArmyScaler is now always fresh
+ // after Eval() returns, including on the NNUE path -- previously
+ // it was left stale (whatever a prior call last set) whenever
+ // Eval() took the NNUE short-circuit, since that return happened
+ // before this block ran at all. Nothing on the NNUE path
+ // previously depended on uArmyScaler being stale vs fresh, so this
+ // is strictly safer, not a behavior change worth gating.
pos->uArmyScaler[BLACK] = pos->uNonPawnMaterial[BLACK] - VALUE_KING;
pos->uArmyScaler[WHITE] = pos->uNonPawnMaterial[WHITE] - VALUE_KING;
pos->uArmyScaler[BLACK] /= VALUE_PAWN;
@@ -7076,6 +7236,143 @@ Return value:
(SystemReadTimeStampCounter() - u64MaterialSetupTimer);
#endif
+ if ((TRUE == g_fUseNnueEval) && (TRUE == g_fNnueWeightsLoaded))
+ {
+ // p98 of |NnueForward() - NnueProbeScore()| per material
+ // bucket. Superseded 2026-10-15: originally derived from a
+ // 91M-sample population over three curated tactical suites
+ // (sd 10) -- replaced same day by a far larger and more
+ // representative 3.09-BILLION-sample population, 16 parallel
+ // shards of 12,500 positions each (200,000 total) from
+ // tests/cross_sectional_200k.ep_ at sd 6. Per-suite tactical
+ // positions are a narrow, biased slice (selected for sharp
+ // tactics); the cross-sectional pool's much broader position
+ // variety surfaced real outlier swings the narrower sample
+ // never saw (max up to 534cp here vs ~472cp before) while
+ // landing close to the same per-bucket numbers overall (mostly
+ // 0-10cp different) -- see docs/NNUE.md's "Lazy-evaluation
+ // tiering plan". Same bucketing as _MaterialBucket (combined
+ // army scaler, width BASE_MARGIN_MAT_WIDTH).
+ static const SCORE iNnueLazyMarginByMatBucket[BASE_MARGIN_MAT_BUCKETS] =
+ {
+ 180, 200, 190, 180, 170, 160, 150, 150,
+ };
+
+#ifdef CALIBRATE_MARGIN_SAFETY
+ // Calibration build: ALWAYS run the real forward pass, never
+ // skip it -- so any candidate gate decision can keep being
+ // measured against ground truth before it's trusted live.
+ // When the gate is enabled, measure against the real chosen
+ // margin (making "exceeded" a meaningful, ongoing validation
+ // signal); when disabled, still measure against an
+ // effectively-infinite sentinel (pure observation, matches
+ // this harness's original pre-gate behavior).
+ SCORE iNnueScore = NnueForward(ctx);
+ if (TRUE == g_fNnueProbeLoaded)
+ {
+ SCORE iProbeScore = NnueProbeScore(ctx);
+ SCORE iSwing = (SCORE)abs((int)iNnueScore - (int)iProbeScore);
+ SCORE iMarginUsed = (TRUE == g_fUseNnueLazyProbe) ?
+ iNnueLazyMarginByMatBucket[_MaterialBucket(pos)] :
+ INFINITY_SCORE;
+ RecordNnueLazyMarginSafetySwing(pos, iSwing, iMarginUsed);
+ }
+ {
+ // Material-only "super lazy" measurement -- no fit, no
+ // probe, just the already-maintained material balance
+ // (same cp units as NnueForward's output, see
+ // RecordNnueSuperLazyMarginSafetySwing). No real margin
+ // chosen yet, so always the infinite sentinel for now.
+ SCORE iSuperLazyScore = pos->iMaterialBalance[pos->uToMove];
+ SCORE iSuperLazySwing =
+ (SCORE)abs((int)iNnueScore - (int)iSuperLazyScore);
+ RecordNnueSuperLazyMarginSafetySwing(pos, iSuperLazySwing, INFINITY_SCORE);
+ }
+ // This branch always returns the real forward pass (never
+ // skips, by design -- see the big comment above), so it's
+ // unconditionally the NNUE_FULL tier for futility-margin
+ // selection purposes.
+ ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_FULL;
+ g_u64NnueFullEvalsByBucket[_MaterialBucket(pos)]++;
+ if (NULL != piPositional)
+ {
+ (*piPositional)[WHITE] = 0;
+ (*piPositional)[BLACK] = 0;
+ }
+ return iNnueScore;
+#else
+ // Production build: if the lazy probe gate is enabled and
+ // confidently resolves the fail-high/fail-low question
+ // relative to [iAlpha, iBeta], skip the real forward pass
+ // entirely -- this is the actual speed win. Otherwise (gate
+ // disabled, probe not loaded, or inconclusive), fall through
+ // to the full pass exactly as before this feature existed.
+ // Super-lazy pre-filter, experimental -- 2026-10-16: cheapest
+ // possible check, pure material balance, no net evaluation at
+ // all. Previously measurement-only (CALIBRATE_MARGIN_SAFETY
+ // above records the real swing but always used an infinite
+ // sentinel margin, i.e. never actually gated). Deliberately
+ // loose/uncalibrated here per the "safety doesn't win games"
+ // experiment -- g_iNnueSuperLazyMargin is a flat runtime
+ // override (see nnueeval.c), not a per-bucket fit.
+ if (TRUE == g_fUseNnueSuperLazy)
+ {
+ SCORE iSuperLazyScore = pos->iMaterialBalance[pos->uToMove];
+
+ if (((iSuperLazyScore - g_iNnueSuperLazyMargin) > iBeta) ||
+ ((iSuperLazyScore + g_iNnueSuperLazyMargin) < iAlpha))
+ {
+ // Feed the same aggregate counters root.c's "Eval exit
+ // breakdown" line reads -- otherwise that line stays
+ // 0/0/0 under NNUE eval since it was only ever wired
+ // to the HCE's own super-lazy/lazy/full exit sites.
+ INC(ctx->sCounters.tree.u64SuperLazyEvals);
+ ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_LAZY;
+ g_u64NnueLazyEvalsByBucket[_MaterialBucket(pos)]++;
+ if (NULL != piPositional)
+ {
+ (*piPositional)[WHITE] = 0;
+ (*piPositional)[BLACK] = 0;
+ }
+ return iSuperLazyScore;
+ }
+ }
+ if ((TRUE == g_fUseNnueLazyProbe) && (TRUE == g_fNnueProbeLoaded))
+ {
+ SCORE iProbeScore = NnueProbeScore(ctx);
+ SCORE iMargin = (g_iNnueLazyMarginOverride >= 0) ?
+ (SCORE)g_iNnueLazyMarginOverride :
+ iNnueLazyMarginByMatBucket[_MaterialBucket(pos)];
+
+ if (((iProbeScore - iMargin) > iBeta) ||
+ ((iProbeScore + iMargin) < iAlpha))
+ {
+ INC(ctx->sCounters.tree.u64LazyEvals);
+ ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_LAZY;
+ g_u64NnueLazyEvalsByBucket[_MaterialBucket(pos)]++;
+ if (NULL != piPositional)
+ {
+ (*piPositional)[WHITE] = 0;
+ (*piPositional)[BLACK] = 0;
+ }
+ return iProbeScore;
+ }
+ }
+ {
+ SCORE iNnueScore = NnueForward(ctx);
+ INC(ctx->sCounters.tree.u64FullEvals);
+ ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_FULL;
+ g_u64NnueFullEvalsByBucket[_MaterialBucket(pos)]++;
+ if (NULL != piPositional)
+ {
+ (*piPositional)[WHITE] = 0;
+ (*piPositional)[BLACK] = 0;
+ }
+ return iNnueScore;
+ }
+#endif
+ }
+
#ifdef DUMP_RESIDUAL_SAMPLES
// Super-lazy's OWN baseline -- material only, before _EvalPawns
// even runs. Separate from iResidualBaselineWhite/Black below
@@ -7178,7 +7475,7 @@ Return value:
// 2026-09-17: no longer flat 0 -- same p50-based typical-
// value estimate as regular lazy, using the super-lazy-
// scoped residual table (material-only baseline). See
- // iKingP50ByAttackerArmyOnly/iSuperLazyResidualP50By-
+ // iKingP50ByAttackerArmy/iSuperLazyResidualP50By-
// OwnArmy (file-scope, above).
ULONG uWhiteBucket = MINU(7, pos->uArmyScaler[WHITE] / 4);
ULONG uBlackBucket = MINU(7, pos->uArmyScaler[BLACK] / 4);
@@ -7520,7 +7817,7 @@ Return value:
// terms (king safety + residual), matching full-eval's
// own iPositional convention (that side's own real
// positional score, not a relative-to-opponent
- // quantity). See iKingP50ByAttackerArmyOnly/
+ // quantity). See iKingP50ByAttackerArmy/
// iResidualP50ByOwnArmy (file-scope, above) for the
// full percentile charts this was derived from.
ULONG uWhiteBucket = MINU(7, pos->uArmyScaler[WHITE] / 4);
@@ -7546,10 +7843,10 @@ Return value:
iSavedLazyScore = iScoreForSideToMove;
iSavedLazyPositional = iAlphaMargin;
iSavedLazyPositionalWhite =
- iKingP50ByAttackerArmyOnly[MINU(7, pos->uArmyScaler[BLACK] / 4)] +
+ iKingP50ByAttackerArmy[MINU(7, pos->uArmyScaler[BLACK] / 4)] +
iResidualP50ByOwnArmy[MINU(7, pos->uArmyScaler[WHITE] / 4)];
iSavedLazyPositionalBlack =
- iKingP50ByAttackerArmyOnly[MINU(7, pos->uArmyScaler[WHITE] / 4)] +
+ iKingP50ByAttackerArmy[MINU(7, pos->uArmyScaler[WHITE] / 4)] +
iResidualP50ByOwnArmy[MINU(7, pos->uArmyScaler[BLACK] / 4)];
#else
#ifdef EVAL_TIME
@@ -7591,10 +7888,10 @@ Return value:
iSavedLazyScore = iScoreForSideToMove;
iSavedLazyPositional = iBetaMargin;
iSavedLazyPositionalWhite =
- iKingP50ByAttackerArmyOnly[MINU(7, pos->uArmyScaler[BLACK] / 4)] +
+ iKingP50ByAttackerArmy[MINU(7, pos->uArmyScaler[BLACK] / 4)] +
iResidualP50ByOwnArmy[MINU(7, pos->uArmyScaler[WHITE] / 4)];
iSavedLazyPositionalBlack =
- iKingP50ByAttackerArmyOnly[MINU(7, pos->uArmyScaler[WHITE] / 4)] +
+ iKingP50ByAttackerArmy[MINU(7, pos->uArmyScaler[WHITE] / 4)] +
iResidualP50ByOwnArmy[MINU(7, pos->uArmyScaler[BLACK] / 4)];
#else
#ifdef EVAL_TIME
@@ -8127,10 +8424,10 @@ Return value:
ULONG uWhiteBucket = MINU(7, pos->uArmyScaler[WHITE] / 4);
ULONG uBlackBucket = MINU(7, pos->uArmyScaler[BLACK] / 4);
(*piPositional)[WHITE] =
- iKingP50ByAttackerArmyOnly[uBlackBucket] +
+ iKingP50ByAttackerArmy[uBlackBucket] +
iSuperLazyResidualP50ByOwnArmy[uWhiteBucket];
(*piPositional)[BLACK] =
- iKingP50ByAttackerArmyOnly[uWhiteBucket] +
+ iKingP50ByAttackerArmy[uWhiteBucket] +
iSuperLazyResidualP50ByOwnArmy[uBlackBucket];
// CALIBRATE_MARGIN_SAFETY builds: the FULL tag set above
// reflects the fallthrough that just ran for measurement
diff --git a/src/eval_tune/build_cross_sectional_all.py b/src/eval_tune/build_cross_sectional_all.py
new file mode 100644
index 0000000..144c87c
--- /dev/null
+++ b/src/eval_tune/build_cross_sectional_all.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+"""Build a larger cross-sectional position pool (successor to
+tests/cross_sectional_200k.ep_) out of the full twic.pgn +
+lichess_elite*.pgn corpus.
+
+Unlike sample_eval_calibration_positions.py's random-byte-offset
+sampling (one position per game, uniform-random ply), this does an
+exhaustive streaming pass over every game and pulls 5 positions per
+qualifying game at fixed fractional points in the game
+(early/early-mid/mid/mid-late/late) -- Scott's own framing. Fractional-
+ply sampling structurally balances game-stage representation (every
+kept game contributes exactly one position to each of the 5 stage
+buckets), which random-ply sampling does not.
+
+Deliberately does NOT filter for quiescence -- these positions are
+`go` roots for the engine's own search (which resolves tactics via its
+own qsearch at every node), not Texel-tuning eval-only positions. See
+sample_eval_calibration_positions.py's own docstring for the same
+reasoning: a cross-sectional pool should look like "all positions a
+real search tree touches", not just the quiet subset.
+
+Ply fractions: 5 samples come from 6 equal segments (1/6..5/6 of the
+game's ply count), not literal 1/5..5/5 -- 5/5 is the final move of
+the game, frequently checkmate/stalemate/resignation-adjacent, which
+`go` can't search meaningfully. Interior boundaries only.
+"""
+import argparse
+import collections
+import io
+import sys
+from pathlib import Path
+
+import chess
+import chess.pgn
+
+STAGE_NAMES = ["early", "early_mid", "mid", "mid_late", "late"]
+FRACTIONS = [1 / 6, 2 / 6, 3 / 6, 4 / 6, 5 / 6]
+
+PIECE_VALUES = {chess.KNIGHT: 3, chess.BISHOP: 3, chess.ROOK: 5, chess.QUEEN: 9}
+
+
+def material_bucket(board):
+ """Same bucketing as eval.c's _MaterialBucket / the calibration
+ script's own copy: combined non-pawn army value (pawn units) / 8,
+ capped at 7 (0 = bare kings, 7 = full army)."""
+ total = 0
+ for pt, v in PIECE_VALUES.items():
+ total += v * len(board.pieces(pt, chess.WHITE))
+ total += v * len(board.pieces(pt, chess.BLACK))
+ return min(7, total // 8)
+
+
+def iter_game_texts(pgn_path):
+ """Stream raw PGN text blocks, one game at a time, without loading
+ the whole file -- these corpora run into the GB range (twic.pgn
+ alone is 3.5GB)."""
+ with open(pgn_path, "r", encoding="utf-8", errors="replace") as f:
+ header_lines = []
+ movetext_lines = []
+ in_headers = True
+ for line in f:
+ if line.startswith("["):
+ if not in_headers and (header_lines or movetext_lines):
+ yield "".join(header_lines) + "\n" + "".join(movetext_lines)
+ header_lines, movetext_lines = [], []
+ header_lines.append(line)
+ in_headers = True
+ elif line.strip() == "":
+ if header_lines and not movetext_lines:
+ in_headers = False
+ else:
+ in_headers = False
+ movetext_lines.append(line)
+ if header_lines or movetext_lines:
+ yield "".join(header_lines) + "\n" + "".join(movetext_lines)
+
+
+def normalized_fen_key(fen):
+ """Board+side-to-move+castling+ep, dropping halfmove/fullmove
+ counters -- used only for dedup accounting, never written out."""
+ parts = fen.split()
+ return " ".join(parts[:4])
+
+
+def process_pgn(path, min_elo, min_ply, stats, seen_keys, out_f):
+ kept_games = 0
+ scanned_games = 0
+ for raw in iter_game_texts(path):
+ scanned_games += 1
+ game = chess.pgn.read_game(io.StringIO(raw))
+ if game is None:
+ continue
+ headers = game.headers
+ try:
+ we = int(headers.get("WhiteElo", "0"))
+ be = int(headers.get("BlackElo", "0"))
+ except ValueError:
+ we = be = 0
+ if min_elo and (we < min_elo or be < min_elo):
+ stats["rejected_elo"] += 1
+ continue
+
+ moves = list(game.mainline_moves())
+ if len(moves) < min_ply:
+ stats["rejected_short"] += 1
+ continue
+
+ result = headers.get("Result", "*")
+ stats["games_by_result"][result] += 1
+ stats["source_games"][path.name] += 1
+ kept_games += 1
+
+ board = game.board()
+ move_idx = 0
+ boundaries = [round(f * len(moves)) for f in FRACTIONS]
+ # de-dup consecutive identical boundaries (very short-but-kept
+ # games can round two fractions to the same ply)
+ seen_boundaries = set()
+ bi = 0
+ for target_ply in boundaries:
+ target_ply = max(1, min(target_ply, len(moves) - 1))
+ while move_idx < target_ply:
+ board.push(moves[move_idx])
+ move_idx += 1
+ stage = STAGE_NAMES[bi]
+ bi += 1
+ if target_ply in seen_boundaries:
+ stats["skipped_duplicate_boundary"] += 1
+ continue
+ seen_boundaries.add(target_ply)
+ if board.is_game_over():
+ stats["skipped_game_over"] += 1
+ continue
+ fen = board.fen()
+ key = normalized_fen_key(fen)
+ stats["stage_total"][stage] += 1
+ stats["material_bucket"][material_bucket(board)] += 1
+ stats["side_to_move"]["white" if board.turn else "black"] += 1
+ if key in seen_keys:
+ stats["duplicate_fens"] += 1
+ stats["stage_duplicates"][stage] += 1
+ continue
+ seen_keys.add(key)
+ out_f.write(f"setboard {fen}\n")
+ out_f.write("go\n")
+ stats["written"] += 1
+ stats["stage_written"][stage] += 1
+
+ stats["scanned_games"] += scanned_games
+ stats["kept_games"] += kept_games
+ return kept_games
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("pgn_paths", nargs="+")
+ ap.add_argument("--min-elo", type=int, default=2000)
+ ap.add_argument("--min-ply", type=int, default=30,
+ help="minimum game length in plies to be sampled "
+ "at all -- filters short draws/miniatures")
+ ap.add_argument("--out", required=True)
+ ap.add_argument("--progress-every", type=int, default=50000)
+ args = ap.parse_args()
+
+ stats = {
+ "scanned_games": 0,
+ "kept_games": 0,
+ "rejected_elo": 0,
+ "rejected_short": 0,
+ "skipped_game_over": 0,
+ "skipped_duplicate_boundary": 0,
+ "duplicate_fens": 0,
+ "written": 0,
+ "games_by_result": collections.Counter(),
+ "source_games": collections.Counter(),
+ "stage_total": collections.Counter(),
+ "stage_written": collections.Counter(),
+ "stage_duplicates": collections.Counter(),
+ "material_bucket": collections.Counter(),
+ "side_to_move": collections.Counter(),
+ }
+ seen_keys = set()
+
+ with open(args.out, "w") as out_f:
+ for p in args.pgn_paths:
+ path = Path(p)
+ print(f"processing {path.name}...", file=sys.stderr)
+ before = stats["kept_games"]
+ process_pgn(path, args.min_elo, args.min_ply, stats, seen_keys, out_f)
+ print(f" kept {stats['kept_games'] - before} games "
+ f"(running total written: {stats['written']})", file=sys.stderr)
+
+ print("\n=== summary ===")
+ print(f"games scanned: {stats['scanned_games']:,}")
+ print(f"games kept: {stats['kept_games']:,}")
+ print(f" rejected (elo): {stats['rejected_elo']:,}")
+ print(f" rejected (too short): {stats['rejected_short']:,}")
+ print(f"positions written: {stats['written']:,}")
+ print(f"duplicate fens skipped: {stats['duplicate_fens']:,} "
+ f"({100.0*stats['duplicate_fens']/max(1,stats['written']+stats['duplicate_fens']):.1f}%)")
+ print(f"game-over boundaries skipped: {stats['skipped_game_over']:,}")
+ print()
+ print("by result:")
+ for r, c in stats["games_by_result"].most_common():
+ print(f" {r:>8}: {c:,}")
+ print()
+ print("by source file:")
+ for s, c in stats["source_games"].most_common():
+ print(f" {s}: {c:,}")
+ print()
+ print("by game stage (written / total-before-dedup):")
+ for stage in STAGE_NAMES:
+ w = stats["stage_written"][stage]
+ t = stats["stage_total"][stage]
+ d = stats["stage_duplicates"][stage]
+ print(f" {stage:>10}: written={w:,} dup_rate={100.0*d/max(1,t):.1f}%")
+ print()
+ print("by material bucket (0=bare kings .. 7=full army):")
+ for b in range(8):
+ print(f" bucket {b}: {stats['material_bucket'][b]:,}")
+ print()
+ print("by side to move:")
+ for k, c in stats["side_to_move"].items():
+ print(f" {k}: {c:,}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/eval_tune/compare_vs_crafty.py b/src/eval_tune/compare_vs_crafty.py
new file mode 100644
index 0000000..318ff87
--- /dev/null
+++ b/src/eval_tune/compare_vs_crafty.py
@@ -0,0 +1,597 @@
+#!/usr/bin/env python3
+"""
+Cross-engine static-eval audit: compare typhoon's `eval` command output
+against a *crowd* of other hand-crafted-eval engines (Crafty, Kiwi, Fruit,
+GNU Chess -- all pre-NNUE, additive-term evals, so genuinely comparable to
+typhoon's own, unlike Stockfish's NNUE), rank the positions where typhoon
+is the outlier against the crowd's median, and attribute the gap back to
+typhoon's own named EVAL_TERM lines (from a build with EVAL_DUMP=1).
+
+Methodology, and the gotchas that cost real time to find (2026-09-2x
+eval audit session):
+
+ - Build typhoon with EVAL_DUMP=1 (see CLAUDE.md's build instructions)
+ to get the per-term breakdown table, not just the "Static eval:"
+ total. The breakdown itself is a SEPARATE command, `dump eval`
+ (EvalTraceReport() in testeval.c) -- it reports on the terms
+ recorded by the *last* Eval() call, so the script sends `eval` then
+ `dump eval` back to back for every position, not `eval` alone.
+
+ - Run typhoon with --hash none. Not an optimization -- a correctness-
+ adjacent perf fix. SetBoardCommand -> PreGameReset(FALSE) ->
+ ClearHashTable() unconditionally, so every single `setboard` with a
+ real --hash size does a full memset of the whole hash table. Over
+ 2000 positions this was the entire difference between ~46s and
+ ~2.2s (21x) for byte-identical output. Always pass --hash none for
+ eval-only batch runs; there is no search happening, so there is
+ nothing to hash.
+
+ - Feed positions via `script <file>` (one `setboard <fen>` / `eval`
+ pair per line), not a single giant --command string. A few thousand
+ positions' worth of inline command text can exceed the shell's/
+ kernel's argv length limits; `script` reads from a file instead and
+ has no such ceiling.
+
+ - Only ONE process per engine for the whole batch -- every engine here
+ correctly returns different, real scores across repeated
+ setboard/eval calls in a single session; none needs restarting per
+ position. (Verified directly this session by feeding
+ /usr/local/bin/crafty three visibly different positions in one
+ stdin session and getting three different totals.)
+
+ - Use /usr/local/bin/crafty (whichever `crafty` resolves to on PATH),
+ NOT ~/programming/chess/crafty-25.2/crafty (formerly ~/crafty/crafty
+ -- that whole directory was moved to this path). The latter is a
+ different, older build (confirmed via md5) that a sibling script
+ here (compare_king_safety_crafty.py) found DOES silently reuse the
+ first position's cached score on a second setboard+score in one
+ session -- that bug is specific to that binary, not a general
+ Crafty limitation, and does not reproduce against
+ /usr/local/bin/crafty. If you ever see suspiciously identical
+ Crafty totals across different positions, check which crafty
+ binary you're actually calling before assuming a new engine bug.
+
+ - Crowd engines each have their own sign convention for the "eval"
+ command's printed number, confirmed empirically (setboard a position
+ with white down a whole knight and check the sign/magnitude), not
+ assumed from reading their source alone:
+ * crafty `score`, kiwi `eval` -- print the score from WHITE's
+ perspective already, no flip needed.
+ * fruit `eval` -- prints from the SIDE-TO-MOVE's perspective
+ (negamax convention: eval.cpp's `eval()` explicitly negates for
+ Black at the very end). Must flip sign when it's Black to move
+ to get a White-perspective number comparable to the others.
+ * gnuchess (gnuchessr) `score` -- ALWAYS prints "(white to
+ move)" and an always-White-relative number, regardless of the
+ loaded position's actual active color. Confirmed empirically:
+ loading the identical position via two xpos files differing
+ only in trailing "w"/"b" produced byte-identical output both
+ times. Root cause (read, not guessed): `xget` -> GetXGame()
+ (dspcom.c) sets the `computer` global from the file's color
+ field but never touches the separate `player` global that
+ ScorePosition()'s perspective and the printed "(%s to move)"
+ label actually key off -- `player` is a zero-initialized global
+ that only search.c's search loop ever updates, so outside of an
+ actual search it just stays at its BSS-zero value (White). Net
+ effect: harmless for this script (means no stm-relative flip is
+ ever needed for gnuchess) but it DOES mean gnuchess's score
+ never reflects tempo/mobility-from-whoever's-actually-on-move
+ nuance -- a known accuracy ceiling on this crowd member, not a
+ bug in this script.
+ Getting a sign convention wrong silently halves the apparent
+ disagreement rate (a sign-flipped engine agrees with the crowd
+ exactly when it should maximally disagree, and vice versa) rather
+ than crashing, so it does not announce itself -- always spot check
+ a lopsided material position per engine before trusting a new one's
+ numbers.
+
+ - Kiwi's own `kiwi` binary in ~/programming/chess/kiwi can be stale
+ relative to its own source (confirmed: mtimes showed the checked-in
+ binary predated winboard.cxx/engine.cxx, and it silently mis-parses
+ `eval` as an illegal `usermove` rather than erroring, because
+ command-table dispatch on prefix match falls through to
+ cmd_OpponentMoves on a lookup miss) -- there was already a correctly
+ rebuilt `kiwi.new` sitting alongside it in that same session, which
+ is what this script actually points at
+ (KIWI_BIN, below). If kiwi's numbers ever look wrong, rebuild with
+ `env OBJDIR=. LD=clang++ CC=clang++ gmake` from ~/programming/chess/kiwi
+ (its Makefile uses GNU-make-only `$(addprefix ...)`, so it must be
+ `gmake`, not BSD `make`; g++ doesn't exist on this box, hence the
+ clang++ overrides) and re-point KIWI_BIN at the fresh binary.
+
+ - gnuchess (gnuchessr, ~/programming/chess/gnuchess/src) has no
+ `setboard`/`position fen` command at all -- the only way to load an
+ arbitrary position is `xget`, an interactive prompt that reads a
+ FILE in gnuchess's old xboard.position.read format (just FEN's
+ piece-placement and active-color fields, one line, no
+ castling/ep/clock -- castling rights are simply lost, an inherent
+ approximation for this crowd member). `_build_gnuchess_script`
+ writes one such tiny file per position under
+ /tmp/typhoon/eval_audit/gnuchess/ rather than trying to reuse one
+ filename, since the position must be fully written before the
+ single stdin-scripted process reaches that `xget` line. Also: the
+ process needs one leading blank line to satisfy its startup
+ "Enter [moves] minutes[:sec]..." time-control prompt before the
+ "Your move is?" loop (where `xget`/`score`/`quit` are valid
+ commands) even begins reading commands -- a script missing that
+ blank line gets its first real command silently swallowed by the
+ time-control prompt instead. Also needs its OWN directory as cwd
+ (same gotcha class as Glaurung/kpk.bin in an earlier version of
+ this crowd): run from elsewhere it prints "NO LANGFILE (file
+ gnuchess.lang not found)" and exits immediately instead of falling
+ back gracefully.
+
+ - Compare typhoon against the crowd's MEDIAN, not vs. any single
+ other engine and not vs. the mean. Median is robust to any one
+ crowd engine being the actual outlier on a given position (e.g. a
+ known-narrow specialization one of them has and the others don't);
+ a mean lets one bad engine drag the "consensus" toward itself.
+ Historically (see the old Crafty-only version of this script)
+ averaging in an NNUE engine (Stockfish) made typhoon look like the
+ outlier for tracking a hand-crafted eval correctly (9 of 10 checked
+ outliers were actually typhoon+Crafty agreeing within ~0.3 pawns
+ with Stockfish alone ~12-15 pawns off) -- same failure mode extends
+ to any structurally-different evaluator diluting the crowd, so keep
+ the crowd restricted to same-family (additive, hand-tuned) evals.
+
+ - This box's kern.ipc.semmni (max SysV semaphore sets) is only 50.
+ Every typhoon process leaks its semaphore set on `kill -9` (a clean
+ `quit`/EOF releases it; a hard kill does not, regardless of the
+ unix.c pthread-mutex/condvar rewrite -- that fixed a different
+ leak). Repeatedly killing hung/timed-out typhoon processes during
+ debugging can silently exhaust the limit; once exhausted, EVERY new
+ typhoon process falls back to a ~1000x-slower 100ms polling loop
+ for input, which looks exactly like a hang or a new performance
+ regression. Check with `ipcs -s | grep $(whoami) | wc -l` before
+ concluding a slowdown is a code bug; clean up with `ipcrm -s <id>`
+ for each leaked entry (or `for id in $(ipcs -s | grep $(whoami) |
+ awk '{print $2}'); do ipcrm -s "$id"; done`).
+
+Usage:
+
+ python3 compare_vs_crafty.py <fen_source> [--limit N] [--top K]
+ [--engines crafty,kiwi,fruit,gnuchess]
+
+ <fen_source> is either a path to an .ep_-style file (lines of
+ "setboard <fen>" / "go", as in ../tests/*.ep_) or a plain file of one
+ FEN per line. Requires a typhoon binary built with EVAL_DUMP=1 at
+ ../typhoon (build it first: `gmake clean && gmake -j5 GENETIC=1
+ PERF_COUNTERS=1 MP=1 SIXTYFOUR=1 EVAL_DUMP=1` from src/), plus
+ whichever crowd engine binaries are selected (all on by default;
+ see ENGINE_SPECS for their expected paths).
+
+Output: prints the top-K disagreements (|typhoon(white) -
+crowd_median(white)|), each crowd engine's own white-perspective score
+for context, and typhoon's largest contributing EVAL_TERM line(s) from
+its own perspective -- a quick first read on which term is driving the
+gap, before digging in by hand.
+"""
+import argparse
+import re
+import statistics
+import subprocess
+import sys
+from pathlib import Path
+
+SRC_DIR = Path(__file__).resolve().parent.parent
+TYPHOON_BIN = SRC_DIR / "typhoon"
+SCRATCH = Path("/tmp/typhoon/eval_audit")
+
+CHESS_DIR = Path.home() / "programming" / "chess"
+
+# Every crowd engine speaks a different protocol/command for "set this
+# position, print static eval." `kind` selects how the position+eval
+# commands are framed; `parse_re` must capture exactly the numeric score
+# (possibly negative) as group 1; `scale` converts its native units to
+# pawns; `stm_relative` marks engines whose printed number is from the
+# SIDE TO MOVE's perspective (needs a sign flip on Black-to-move
+# positions) rather than already being White-relative.
+ENGINE_SPECS = {
+ "crafty": dict(
+ binary="crafty", cwd=None, kind="xboard",
+ startup="ponder off\n", pos_fmt="setboard {fen}\n", eval_cmd="score",
+ parse_re=r'^total\.+\s+([+-]?\d+\.\d+)', scale=1.0, stm_relative=False,
+ ),
+ "kiwi": dict(
+ binary=str(CHESS_DIR / "kiwi" / "kiwi.new"), cwd=None, kind="xboard",
+ startup="", pos_fmt="setboard {fen}\n", eval_cmd="eval",
+ parse_re=r'^Static eval\s*=\s*([+-]?\d+)', scale=0.01, stm_relative=False,
+ ),
+ "fruit": dict(
+ binary=str(CHESS_DIR / "fruit" / "fruit"), cwd=None, kind="uci",
+ startup="", pos_fmt="position fen {fen}\n", eval_cmd="eval",
+ parse_re=r'info string eval\s+([+-]?\d+)', scale=0.01, stm_relative=True,
+ ),
+ "gnuchess": dict(
+ binary=str(CHESS_DIR / "gnuchess" / "src" / "gnuchessr"),
+ cwd=str(CHESS_DIR / "gnuchess" / "src"),
+ kind="gnuchess", scale=0.01, stm_relative=False,
+ # Not anchored at line-start: gnuchess's "Your move is? " prompt
+ # precedes "Static score..." on the SAME line, unlike every
+ # other engine here which prints its own line cleanly.
+ parse_re=r'Static score \(\w+ to move\):\s*([+-]?\d+)',
+ ),
+}
+
+
+def extract_fens(path: Path) -> list[str]:
+ fens = []
+ for line in path.read_text().splitlines():
+ line = line.strip()
+ if line.startswith("setboard "):
+ fens.append(line[len("setboard "):].strip())
+ elif line and not line.startswith(("go", "#")) and " " in line and "/" in line:
+ # plain FEN file
+ fens.append(line)
+ return fens
+
+
+def run_typhoon_eval_dump(fens: list[str]) -> str:
+ SCRATCH.mkdir(parents=True, exist_ok=True)
+ script_path = SCRATCH / "typhoon_script.txt"
+ out_path = SCRATCH / "typhoon_out.txt"
+ with open(script_path, "w") as f:
+ for fen in fens:
+ # `eval` alone only prints the "Static eval:" total; the
+ # per-term "--- WHITE ... BLACK" breakdown table comes from
+ # the separate `dump eval` command (EvalTraceReport(),
+ # testeval.c), which reports on the terms from the *last*
+ # Eval() call -- so it must follow `eval` for the same
+ # position, not stand alone.
+ f.write(f"setboard {fen}\neval\ndump eval\n")
+ cmd = [str(TYPHOON_BIN), "--cpus", "1", "--hash", "none",
+ "--logfile", "-", "--batch", "--command",
+ f"force; book name /nonexistent.book.bin; script {script_path}"]
+ with open(out_path, "w") as out:
+ subprocess.run(cmd, stdout=out, stderr=subprocess.STDOUT, cwd=SRC_DIR,
+ timeout=max(60, len(fens) // 20))
+ return out_path.read_text()
+
+
+def run_crowd_engine(name: str, fens: list[str]) -> str:
+ spec = ENGINE_SPECS[name]
+ if spec["kind"] == "gnuchess":
+ script = _build_gnuchess_script(fens)
+ else:
+ script = spec["startup"]
+ for fen in fens:
+ script += spec["pos_fmt"].format(fen=fen) + spec["eval_cmd"] + "\n"
+ script += "quit\n"
+ p = subprocess.run([spec["binary"]], input=script, capture_output=True,
+ text=True, cwd=spec["cwd"], timeout=max(60, len(fens) // 20))
+ return p.stdout
+
+
+def _build_gnuchess_script(fens: list[str]) -> str:
+ """gnuchess (gnuchessr) has no `setboard`/`position fen` -- the only
+ way to load an arbitrary position is `xget`, which prompts for a
+ filename and reads a file in its own old xboard.position.read
+ format: just FEN's first two fields (piece placement + active
+ color), one per file, no castling/ep/clock info. Also: a leading
+ blank line is required first to accept the default time control
+ prompt before the "Your move is?" command loop (where `xget` is a
+ valid command) even starts accepting input."""
+ gc_dir = SCRATCH / "gnuchess"
+ gc_dir.mkdir(parents=True, exist_ok=True)
+ script = "\n"
+ for i, fen in enumerate(fens):
+ parts = fen.split()
+ pos_path = gc_dir / f"pos_{i}.xpos"
+ pos_path.write_text(f"{parts[0]} {parts[1]}\n")
+ script += f"xget\n{pos_path}\nscore\n"
+ script += "quit\n"
+ return script
+
+
+def parse_typhoon(text: str, n: int):
+ evals = [float(x) for x in
+ re.findall(r'^Static eval:\s*([+-]?\d+\.\d+)', text, re.M)]
+ assert len(evals) == n, f"expected {n} typhoon evals, got {len(evals)}"
+ return evals
+
+
+def parse_crowd_engine(name: str, text: str, stm: list[str], n: int):
+ """Returns n scores in pawns, from WHITE's perspective."""
+ spec = ENGINE_SPECS[name]
+ raw = [float(x) for x in re.findall(spec["parse_re"], text, re.M)]
+ assert len(raw) == n, (
+ f"expected {n} {name} evals, got {len(raw)} -- protocol/parse_re "
+ f"mismatch, or {name}'s binary crashed/changed output format")
+ white = []
+ for val, side in zip(raw, stm):
+ v = val * spec["scale"]
+ if spec["stm_relative"] and side == 'b':
+ v = -v
+ white.append(v)
+ return white
+
+
+LABEL_RE_CACHE = {}
+
+
+def term_labels_from_eval_c() -> list[str]:
+ """Pull every EVAL_TERM(...) call's label string directly out of
+ eval.c, so this stays in sync with the source without needing a
+ separately-maintained list."""
+ src = (SRC_DIR / "eval.c").read_text()
+ labels = set()
+ for m in re.finditer(r'EVAL_TERM\(', src):
+ start = m.end()
+ depth = 1
+ i = start
+ while depth > 0 and i < len(src):
+ if src[i] == '(':
+ depth += 1
+ elif src[i] == ')':
+ depth -= 1
+ i += 1
+ # Allow escaped quotes/backslashes inside the string literal
+ # (e.g. eval.c's own "in/out of \"action\"" label) -- a naive
+ # [^"]* stops at the first escaped quote and can pick up a
+ # spurious empty trailing match instead of the real label.
+ strs = re.findall(r'"((?:[^"\\]|\\.)*)"', src[start:i - 1])
+ if strs:
+ labels.add(strs[-1].replace('\\"', '"'))
+ return sorted(labels, key=len, reverse=True)
+
+
+def get_label_re():
+ if 're' not in LABEL_RE_CACHE:
+ labels = term_labels_from_eval_c()
+ LABEL_RE_CACHE['re'] = re.compile(
+ '(' + '|'.join(re.escape(l) for l in labels) + r')\s*(-|[a-h][1-8])?\s+(-?\d+)')
+ return LABEL_RE_CACHE['re']
+
+
+def parse_terms_for_block(block: str):
+ """block is the text between '--- WHITE' and 'Static eval:' for one
+ position. Returns {label: contribution_to_WHITE} (white/black columns
+ split at column 40, matching the dump's fixed layout). Always
+ White-relative regardless of side to move -- each column is that
+ color's own raw contribution, so White's column adds and Black's
+ subtracts from the White-relative total, independent of stm. (Do not
+ key the sign off stm: that gives the contribution to the side TO
+ MOVE instead, which flips sign on every Black-to-move position and
+ silently destroys any correlation against a White-relative residual.)
+ """
+ label_re = get_label_re()
+ contrib = {}
+ for line in block.splitlines():
+ for half, color in ((line[:40], 'w'), (line[40:], 'b')):
+ for m in label_re.finditer(half):
+ label, val = m.group(1), int(m.group(3))
+ sign = 1 if color == 'w' else -1
+ contrib[label] = contrib.get(label, 0) + sign * val
+ return contrib
+
+
+def split_typhoon_blocks(text: str) -> list[str]:
+ # Each block runs from its own "--- WHITE" header up to (but not
+ # including) the NEXT position's leading board-dump text -- trim at
+ # this block's own "Static eval:" line, or the raw split would
+ # include the following position's FEN/hash/board-art/"Material:"
+ # preamble as trailing garbage, which can spuriously match the term
+ # regex (large hex-signature/material digit runs sitting near label
+ # text by coincidence).
+ parts = re.split(r'(?=^--- WHITE)', text, flags=re.M)
+ blocks = []
+ for p in parts:
+ if not p.startswith('--- WHITE'):
+ continue
+ m = re.search(r'^Static eval:.*$', p, flags=re.M)
+ blocks.append(p[:m.end()] if m else p)
+ return blocks
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("fen_source")
+ ap.add_argument("--limit", type=int, default=2000,
+ help="max positions to sample (default 2000)")
+ ap.add_argument("--top", type=int, default=20,
+ help="how many biggest disagreements to print")
+ ap.add_argument("--engines", default=",".join(ENGINE_SPECS),
+ help=f"comma-separated crowd engines to use, from "
+ f"{{{','.join(ENGINE_SPECS)}}} (default: all)")
+ ap.add_argument("--sign-deadband", type=float, default=0.5,
+ help="pawns; both |typhoon| and |crowd median| (or "
+ "unanimous crowd value) must clear this to count "
+ "as a real sign disagreement, not ~0 noise "
+ "(default 0.5)")
+ ap.add_argument("--spread-floor", type=float, default=0.25,
+ help="pawns; floor on crowd stdev used when ranking "
+ "gap/spread so a near-unanimous crowd doesn't "
+ "produce a division-by-near-zero blowup "
+ "(default 0.25)")
+ args = ap.parse_args()
+
+ engines = [e.strip() for e in args.engines.split(",") if e.strip()]
+ for e in engines:
+ if e not in ENGINE_SPECS:
+ sys.exit(f"unknown engine {e!r} -- choose from {list(ENGINE_SPECS)}")
+ if not Path(ENGINE_SPECS[e]["binary"]).is_absolute() and \
+ ENGINE_SPECS[e]["cwd"] is None:
+ continue # resolved via PATH (e.g. crafty)
+ if not TYPHOON_BIN.exists():
+ sys.exit(f"missing {TYPHOON_BIN} -- build with EVAL_DUMP=1 first "
+ "(see this file's docstring)")
+
+ fens = extract_fens(Path(args.fen_source))[:args.limit]
+ stm = [f.split()[1] for f in fens]
+ print(f"{len(fens)} positions, crowd = {engines}", file=sys.stderr)
+
+ ty_text = run_typhoon_eval_dump(fens)
+ ty_evals = parse_typhoon(ty_text, len(fens))
+ ty_white = [e if s == 'w' else -e for e, s in zip(ty_evals, stm)]
+
+ crowd_white = {}
+ for name in engines:
+ text = run_crowd_engine(name, fens)
+ crowd_white[name] = parse_crowd_engine(name, text, stm, len(fens))
+
+ blocks = split_typhoon_blocks(ty_text)
+ assert len(blocks) == len(fens), (
+ f"{len(blocks)} eval-dump blocks vs {len(fens)} positions -- "
+ "did the typhoon binary actually have EVAL_DUMP=1?")
+ contribs = [parse_terms_for_block(b) for b in blocks]
+
+ n = len(fens)
+ per_pos = [] # (median, stdev, residual, is_extreme, extreme_dir)
+ for i in range(n):
+ crowd_vals = [crowd_white[name][i] for name in engines]
+ median = statistics.median(crowd_vals)
+ stdev = statistics.pstdev(crowd_vals) if len(crowd_vals) > 1 else 0.0
+ residual = ty_white[i] - median
+ all_vals = crowd_vals + [ty_white[i]]
+ is_max = ty_white[i] == max(all_vals) and all_vals.count(ty_white[i]) == 1
+ is_min = ty_white[i] == min(all_vals) and all_vals.count(ty_white[i]) == 1
+ per_pos.append((median, stdev, residual, is_max, is_min))
+
+ def print_position(i, extra=""):
+ median = per_pos[i][0]
+ crowd_str = " ".join(f"{name}={crowd_white[name][i]:+6.2f}"
+ for name in engines)
+ print(f"{extra}ty(w)={ty_white[i]:+6.2f} median(w)={median:+6.2f} "
+ f"{crowd_str} {fens[i]}")
+ top_terms = sorted(contribs[i].items(), key=lambda kv: -abs(kv[1]))[:4]
+ for label, val in top_terms:
+ print(f" {label:38s} {val:+5d}")
+
+ # --- 1. Raw magnitude disagreements (original behavior) -------------
+ diffs = sorted(range(n), key=lambda i: -abs(per_pos[i][2]))
+ print(f"\n=== Top {args.top} disagreements "
+ f"(|typhoon(white) - crowd_median(white)|) ===\n")
+ for i in diffs[:args.top]:
+ print_position(i, extra=f"diff={per_pos[i][2]:+7.2f} ")
+
+ # --- 2. Gap normalized by crowd spread -------------------------------
+ # A raw pawn gap conflates "typhoon is wrong" with "this position is
+ # just wild and the whole crowd disagrees with itself." Dividing by
+ # the crowd's own stdev (floored, so a near-unanimous crowd doesn't
+ # blow up the ratio) surfaces cases where typhoon breaks from an
+ # otherwise-tight consensus, even by a modest raw amount.
+ spread_ranked = sorted(
+ range(n), key=lambda i: -abs(per_pos[i][2]) / max(per_pos[i][1], args.spread_floor))
+ print(f"\n=== Top {args.top} disagreements normalized by crowd spread "
+ f"(|residual| / max(crowd_stdev, {args.spread_floor})) ===\n")
+ for i in spread_ranked[:args.top]:
+ ratio = abs(per_pos[i][2]) / max(per_pos[i][1], args.spread_floor)
+ print_position(i, extra=f"ratio={ratio:6.2f} stdev={per_pos[i][1]:5.2f} ")
+
+ # --- 3. Typhoon as the lone extreme value ----------------------------
+ extremes = [i for i in range(n) if per_pos[i][3] or per_pos[i][4]]
+ extremes.sort(key=lambda i: -abs(per_pos[i][2]))
+ print(f"\n=== Positions where typhoon is the strict min or max of "
+ f"typhoon+crowd ({len(extremes)} of {n}) ===\n")
+ for i in extremes[:args.top]:
+ direction = "HIGHEST" if per_pos[i][3] else "LOWEST"
+ print_position(i, extra=f"typhoon is {direction} ")
+
+ # --- 4. Sign disagreements -------------------------------------------
+ db = args.sign_deadband
+ vs_median, vs_unanimous = [], []
+ for i in range(n):
+ ty, median = ty_white[i], per_pos[i][0]
+ if abs(ty) >= db and abs(median) >= db and (ty > 0) != (median > 0):
+ vs_median.append(i)
+ crowd_vals = [crowd_white[name][i] for name in engines]
+ if all(abs(v) >= db for v in crowd_vals) and \
+ len({v > 0 for v in crowd_vals}) == 1:
+ crowd_sign_positive = crowd_vals[0] > 0
+ if abs(ty) >= db and (ty > 0) != crowd_sign_positive:
+ vs_unanimous.append(i)
+ print(f"\n=== Typhoon disagrees with crowd MEDIAN's sign "
+ f"(both sides >= {db} pawns) -- {len(vs_median)} positions ===\n")
+ for i in vs_median[:args.top]:
+ print_position(i)
+ print(f"\n=== Typhoon disagrees with a UNANIMOUS crowd's sign "
+ f"(every engine agrees, typhoon alone doesn't; >= {db} pawns) "
+ f"-- {len(vs_unanimous)} positions ===\n")
+ for i in vs_unanimous[:args.top]:
+ print_position(i)
+ if "gnuchess" in engines and len(engines) > 1:
+ others = [e for e in engines if e != "gnuchess"]
+ vs_unanimous_no_gc = []
+ for i in range(n):
+ vals = [crowd_white[name][i] for name in others]
+ if all(abs(v) >= db for v in vals) and len({v > 0 for v in vals}) == 1:
+ if abs(ty_white[i]) >= db and (ty_white[i] > 0) != (vals[0] > 0):
+ vs_unanimous_no_gc.append(i)
+ print(f"\n (gnuchess drops castling rights and ignores stm; "
+ f"same check EXCLUDING gnuchess: "
+ f"{len(vs_unanimous_no_gc)} positions)")
+
+ # --- 5. Per-term correlation with the White-relative residual --------
+ # For each EVAL_TERM label, correlate its own signed White-relative
+ # contribution against ty_white - crowd_median across the whole
+ # sample. A term whose contribution consistently tracks the residual
+ # is a candidate for being over- or under-weighted relative to the
+ # crowd's implicit consensus; a term with no correlation isn't
+ # driving the disagreement even if it's large in any one position.
+ residuals = [p[2] for p in per_pos]
+ all_labels = set()
+ for c in contribs:
+ all_labels.update(c.keys())
+ term_stats = []
+ for label in all_labels:
+ vals = [contribs[i].get(label, 0) for i in range(n)]
+ if len(set(vals)) < 2:
+ continue # constant (e.g. always zero) -- correlation undefined
+ try:
+ corr = statistics.correlation(vals, residuals)
+ except statistics.StatisticsError:
+ continue
+ mean_abs = sum(abs(v) for v in vals) / n
+ term_stats.append((label, corr, mean_abs, corr * mean_abs))
+ term_stats.sort(key=lambda t: -abs(t[3]))
+ print(f"\n=== EVAL_TERM correlation with (typhoon - crowd_median), "
+ f"whole sample (n={n}) ===")
+ print("(positive corr = term grows typhoon's score in the same "
+ "direction as its disagreement with the crowd -- a candidate "
+ "for being over-weighted; negative = under-weighted)\n")
+ for label, corr, mean_abs, score in term_stats[:20]:
+ print(f" {label:38s} corr={corr:+.3f} mean|contrib|={mean_abs:6.2f} "
+ f"score={score:+.3f}")
+
+ # --- 6. Global directional bias ---------------------------------------
+ mean_r = statistics.mean(residuals)
+ median_r = statistics.median(residuals)
+ w_res = [residuals[i] for i in range(n) if stm[i] == 'w']
+ b_res = [residuals[i] for i in range(n) if stm[i] == 'b']
+ print(f"\n=== Global bias: typhoon(white) - crowd_median(white) ===\n")
+ print(f" overall: mean={mean_r:+.3f} median={median_r:+.3f} n={n}")
+ if w_res:
+ print(f" white to move: mean={statistics.mean(w_res):+.3f} "
+ f"median={statistics.median(w_res):+.3f} n={len(w_res)}")
+ if b_res:
+ print(f" black to move: mean={statistics.mean(b_res):+.3f} "
+ f"median={statistics.median(b_res):+.3f} n={len(b_res)}")
+ print(" (a nonzero overall mean/median with the same sign on "
+ "both stm splits points at a constant-ish term -- tempo, draw "
+ "score, a global scale factor -- rather than a positional one)")
+
+ # --- 7. Residual by material phase ------------------------------------
+ # Cheap, self-contained phase proxy from the FEN alone (not typhoon's
+ # own internal phase bucketing) so this works even without EVAL_DUMP
+ # internals: sum of N=3/B=3/R=5/Q=9 for both sides, pawns/kings
+ # excluded. Eval bugs are usually phase-local, and a flat top-K list
+ # hides that.
+ piece_val = {'n': 3, 'b': 3, 'r': 5, 'q': 9}
+ phases = []
+ for fen in fens:
+ board = fen.split()[0]
+ phases.append(sum(piece_val.get(ch.lower(), 0) for ch in board if ch.isalpha()))
+ buckets = [(0, 12, "endgame (<=12)"), (13, 26, "middlegame (13-26)"),
+ (27, 999, "opening/near-full (>=27)")]
+ print(f"\n=== Residual by material phase (non-pawn point count, both sides) ===\n")
+ for lo, hi, name in buckets:
+ idxs = [i for i in range(n) if lo <= phases[i] <= hi]
+ if not idxs:
+ continue
+ r = [residuals[i] for i in idxs]
+ print(f" {name:28s} mean={statistics.mean(r):+.3f} "
+ f"median={statistics.median(r):+.3f} n={len(idxs)}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/eval_tune/match_play.py b/src/eval_tune/match_play.py
index 5033cad..34a7917 100755
--- a/src/eval_tune/match_play.py
+++ b/src/eval_tune/match_play.py
@@ -96,7 +96,13 @@ _FIRST_MOVE_BETA_RE = re.compile(
# count, (4) the rest of the line: the SAN PV text (WalkPV, util.c), or
# "FL --" for a fail-low line (no PV -- the move that failed low isn't
# necessarily the move eventually played) or "<SAN> ++" for a fail-high.
-_ITER_RE = re.compile(r"^\s*(\d+)[+-]?\s+(-?\d+)\s+\S+\s+(\d+)\s*(.*)$")
+# 2026-10-17: score group widened from (-?\d+) to ([+-]?\d+) -- typhoon
+# itself never prints a leading "+" on a non-negative score (bare
+# digits), but a UCI engine driven through PolyGlot's xboard adapter
+# (e.g. Stockfish, see stockfish_xboard) DOES print an explicit "+"
+# ("23 +28 198 700479 e4 c5"), which the old pattern rejected outright
+# -- confirmed via direct regex test before this fix, not assumed.
+_ITER_RE = re.compile(r"^\s*(\d+)[+-]?\s+([+-]?\d+)\s+\S+\s+(\d+)\s*(.*)$")
# Reading PostMoveSearchReport's trailing block used to stop either at a
# bounded line count or at gamelist.c's "_white(move N)_"/"_black(move N)_"
# prompt -- both unreliable: the block's line count depends on whether the
@@ -307,6 +313,13 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
white_stats = EngineStats()
black_stats = EngineStats()
white_last_fmb = black_last_fmb = None
+ # Tracked across the whole engine-interaction block below so the
+ # BrokenPipeError handler (see try/except wrapping it) can report
+ # which side/ply was in flight even when the pipe dies on a write
+ # that isn't inside the per-ply move loop (e.g. one of the startup
+ # sends, or a time-control send before the first ply).
+ ply_idx = -1
+ white_to_move = None
# Default Engine() spawns with NO --hash/--cpus at all, which means
# a tiny 65536-entry hash table (main.c's memset-zero default) --
# far smaller than the 256m used everywhere else in this project's
@@ -342,7 +355,8 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
# and the default behavior of a harness in active use should not
# change silently underneath a running experiment.
startup = ["book name /none",
- "set ThinkOnOpponentsTime false"]
+ "set ThinkOnOpponentsTime false",
+ "set PostThrottle 0.0"]
if post:
startup.append("post")
for command in startup:
@@ -416,6 +430,7 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
tail = im.group(4).strip()
if (last_iter_depth is not None and
last_iter_nodes is not None and
+ last_iter_nodes > 0 and
iter_nodes > last_iter_nodes and
iter_depth > last_iter_depth):
iter_log_ratio_sum += (math.log(iter_nodes / last_iter_nodes)
@@ -434,10 +449,12 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
last_eval_score = iter_score
last_eval_pv = tail[:-3].rstrip() if tail.endswith(" ++") else tail
+ abort_reason = None
for _ in range(400):
line = mover.stdout.readline()
if not line:
_dbg(tag, "EOF while scanning for move line")
+ abort_reason = "eof"
break
_dbg(tag, f"read(move-scan): {line!r}")
_note_iter(line)
@@ -445,15 +462,67 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
# Engine resigned instead of moving -- root.c prints
# this and returns without ever printing "move ...",
# so stop reading now rather than blocking forever.
+ # This IS a legitimate decisive result (unlike the
+ # other abort_reason cases below) -- resignation is a
+ # real chess outcome, not a harness/engine failure.
+ abort_reason = "resign"
break
m = _MOVE_RE.match(line)
if m:
- move_uci = m.group(1)
+ # root.c's "move ..." line is xboard/ICS-style
+ # (MoveToIcs, ics.c), which prints the promotion
+ # piece UPPERCASE ("e7e8Q") -- correct for that
+ # protocol, but chess.Move.from_uci() below enforces
+ # strict lowercase UCI and raises ValueError on it.
+ # Unlowered, EVERY promotion move got misclassified
+ # as illegal and the game aborted right as the
+ # promoting side (almost always the winning side in
+ # a long endgame) was about to seal the win -- this
+ # was the actual root cause of the ~47%-corrupted
+ # self-play corpus (docs/NNUE.md, 2026-10-17), not an
+ # engine stall/crash. Squares/files are already
+ # lowercase, so a blanket .lower() is safe here.
+ move_uci = m.group(1).lower()
break
+ else:
+ # Read 400 lines without ever seeing "move ..." (and
+ # without EOF or resignation either) -- most likely an
+ # extremely verbose iteration output (deep search in a
+ # sparse endgame can print far more than 400 lines for a
+ # single move) exceeding this loop's hard cap while the
+ # engine is still alive and about to answer.
+ abort_reason = "line-cap-exceeded"
if move_uci is None:
- # side to move had nothing to play -- treat as loss for mover
- result = 0.0 if white_to_move else 1.0
- return result, moves, move_evals, white_stats, black_stats
+ if abort_reason == "resign":
+ # side to move resigned -- legitimate loss for mover
+ result = 0.0 if white_to_move else 1.0
+ return (result, moves, move_evals, white_stats,
+ black_stats, None)
+ # 2026-10-17: a real bug was found producing exactly this
+ # path for a non-resign reason (eof / line-cap) on a
+ # position where the mover was completely winning --
+ # resignation was disabled for this corpus, so there is
+ # no legitimate explanation for move_uci staying None
+ # other than resign. The OLD code silently recorded this
+ # as an ordinary loss for the mover, which corrupted
+ # ~47% of a self-play training corpus with inverted
+ # results (see docs/NNUE.md). Never do that again: signal
+ # the caller with an aborted (result=None) game instead
+ # of guessing a winner, and capture enough context here
+ # to investigate later without needing to reproduce it
+ # live.
+ exit_code = mover.proc.poll()
+ abort_info = {
+ "reason": abort_reason,
+ "mover": "white" if white_to_move else "black",
+ "ply": ply_idx,
+ "fen_before_ply": board.fen(),
+ "mover_exit_code": exit_code,
+ "mover_alive": exit_code is None,
+ }
+ _dbg(tag, f"ABORT (not scored): {abort_info}")
+ return (None, moves, move_evals, white_stats, black_stats,
+ abort_info)
# PostMoveSearchReport's stats block trails the "move" line
# (confirmed against root.c and real output), not the other
@@ -512,14 +581,32 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
else:
black_last_fmb = ply_fmb
+ move = None
try:
move = chess.Move.from_uci(move_uci)
- if move not in board.legal_moves:
- result = 0.0 if white_to_move else 1.0
- return result, moves, move_evals, white_stats, black_stats
+ illegal = move not in board.legal_moves
except ValueError:
- result = 0.0 if white_to_move else 1.0
- return result, moves, move_evals, white_stats, black_stats
+ illegal = True
+ if illegal:
+ # The engine printed a "move ..." line, but it wasn't a
+ # legal move -- same reasoning as the move_uci-is-None
+ # case above: a real engine bug (producing garbage)
+ # deserves the same abort-not-score treatment, not a
+ # confidently-recorded loss with no diagnostic trail.
+ exit_code = mover.proc.poll()
+ abort_info = {
+ "reason": "illegal-move",
+ "mover": "white" if white_to_move else "black",
+ "ply": ply_idx,
+ "fen_before_ply": board.fen(),
+ "illegal_move_uci": move_uci,
+ "mover_exit_code": exit_code,
+ "mover_alive": exit_code is None,
+ }
+ _dbg(tag, f"ABORT (not scored): {abort_info}")
+ return (None, moves, move_evals, white_stats, black_stats,
+ abort_info)
+ assert move is not None # illegal is False only when the try succeeded
board.push(move)
moves.append(move_uci)
score_white = None
@@ -536,7 +623,7 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
result = 0.5
else:
result = 1.0 if outcome.winner == chess.WHITE else 0.0
- return result, moves, move_evals, white_stats, black_stats
+ return result, moves, move_evals, white_stats, black_stats, None
def _format_eval_comment(score_white, pv, depth):
@@ -576,7 +663,15 @@ def build_pgn_game(start_fen, moves, result_white, white_name, black_name,
game.headers["FEN"] = start_fen
game.headers["Date"] = f"{date.today()}"
game.setup(board)
- result_str = {1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white]
+ # result_white is None for an ABORTED game (play_one_game's mover
+ # failed to produce a legal move for a non-resign reason -- see its
+ # abort_info return value) -- write a standard "unknown result" PGN
+ # tag rather than guessing a winner. fit_k_and_label.py's
+ # result_to_white_score() already treats "*" as "unknown, skip this
+ # game" for training purposes, so this alone keeps an aborted game
+ # out of any future training corpus without touching that script.
+ result_str = ({1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white]
+ if result_white is not None else "*")
game.headers["Result"] = result_str
node = game
for i, move_uci in enumerate(moves):
@@ -888,6 +983,48 @@ def main():
"Without this, local_ebf reads 0.000 -- the "
"engines never print those lines because "
"Engine.__init__ sends 'nopost'.")
+ ap.add_argument("--candidate-name", default="candidate",
+ help="PGN White/Black name recorded for the "
+ "candidate engine (default: 'candidate'). "
+ "2026-10-16: set this to something identifying "
+ "the eval/config actually used (e.g. "
+ "'candidate_nnue_loose') whenever the engine "
+ "isn't a plain HCE build -- self_play_games.pgn "
+ "is reused as NNUE training data later, and a "
+ "generic 'candidate' name makes it impossible "
+ "to tell which games came from a differently-"
+ "calibrated evaluator (whose scores shouldn't "
+ "be trusted as training labels) without "
+ "externally remembering run dates.")
+ ap.add_argument("--baseline-name", default="baseline",
+ help="PGN White/Black name recorded for the "
+ "baseline/head engine (default: 'baseline'); "
+ "same reasoning as --candidate-name.")
+ ap.add_argument("--no-color-swap", action="store_true",
+ help="play each sampled opening ONCE instead of "
+ "twice with colors swapped between head/"
+ "candidate. The swap exists to cancel out "
+ "color-dependent FIRST-MOVE ADVANTAGE in an "
+ "Elo-gating match; it is pure waste -- a byte-"
+ "for-byte duplicate game, just relabeled -- "
+ "whenever head_engine and candidate_engine are "
+ "the SAME binary (as every self-play "
+ "training-data run via "
+ "hce_training_corpus.sh is), because search "
+ "here is deterministic (--cpus 1) given a "
+ "fixed FEN+side-to-move, so swapping which "
+ "name is 'white'/'black' changes no move "
+ "actually played. Confirmed empirically "
+ "2026-10-16: consecutive game pairs in "
+ "nnue_self_play_training_data.pgn were exact "
+ "duplicates (identical FEN, identical move "
+ "list). Use this flag for self-play corpus "
+ "generation -- sample --games distinct "
+ "openings and play each once, rather than "
+ "--games/2 openings played twice -- and keep "
+ "the swap (the default) for real head-vs-"
+ "candidate Elo gating, where two different "
+ "binaries need it to cancel color bias.")
ap.add_argument("--max-plies", type=int, default=200)
ap.add_argument("--scratch", default="/usr/local/tmp/typhoon_match")
ap.add_argument("--log", default="match_history.log")
@@ -968,13 +1105,16 @@ def main():
head_engine = str(Path(args.head_engine).resolve())
candidate_engine = str(Path(args.candidate_engine).resolve())
- n_pairs = args.games // 2
- openings = sample_openings(args.pgn, n_pairs, seed=args.opening_seed)
- n_games = len(openings) * 2
+ if args.no_color_swap:
+ n_openings = args.games
+ else:
+ n_openings = args.games // 2
+ openings = sample_openings(args.pgn, n_openings, seed=args.opening_seed)
jobs = []
for fen in openings:
- for candidate_is_white in (True, False):
+ seats = (True,) if args.no_color_swap else (True, False)
+ for candidate_is_white in seats:
i = len(jobs)
white_engine = candidate_engine if candidate_is_white else head_engine
black_engine = head_engine if candidate_is_white else candidate_engine
@@ -982,6 +1122,7 @@ def main():
candidate_points = 0.0
wins = draws = losses = 0
+ aborted = 0
sprt = Sprt(args.elo0, args.elo1, args.alpha, args.beta) if args.sprt else None
sprt_decision = None
candidate_stats = EngineStats()
@@ -1002,22 +1143,56 @@ def main():
if args.sd is None and head_time != cand_time:
assert ((white_time == cand_time) == candidate_is_white), \
"time-odds seat mapping inverted"
- result_white, moves, move_evals, white_stats, black_stats = play_one_game(
- white_engine, black_engine, f"{args.scratch}/g{i}", fen,
- depth=args.sd,
- white_time_sec=white_time, black_time_sec=black_time,
- max_plies=args.max_plies, post=args.post,
- )
+ try:
+ result_white, moves, move_evals, white_stats, black_stats, abort_info = (
+ play_one_game(
+ white_engine, black_engine, f"{args.scratch}/g{i}", fen,
+ depth=args.sd,
+ white_time_sec=white_time, black_time_sec=black_time,
+ max_plies=args.max_plies, post=args.post,
+ )
+ )
+ except (BrokenPipeError, OSError) as exc:
+ # One of the two engine subprocesses died mid-game (crashed,
+ # was OOM-killed, etc.) -- play_one_game's own abort_info
+ # machinery only covers READ-side failures (EOF/line-cap
+ # while scanning for a move); a WRITE to an already-dead
+ # process's stdin raises BrokenPipeError straight out of
+ # Engine._send instead, uncaught anywhere below this. Left
+ # unhandled, that exception propagates out of this worker
+ # thread's run() through the Future and kills the entire
+ # match (every other in-flight/queued game lost) over a
+ # single flaky engine process -- confirmed live against a
+ # polyglot/Stockfish side via stockfish_xboard. Treat it the
+ # same way as any other abort: no result guessed, game
+ # skipped, run continues.
+ result_white = None
+ moves, move_evals = [], []
+ white_stats, black_stats = EngineStats(), EngineStats()
+ abort_info = {
+ "reason": "engine-pipe-broken",
+ "exception": f"{type(exc).__name__}: {exc}",
+ }
game_elapsed = time.monotonic() - game_start
- candidate_result = result_white if candidate_is_white else (1.0 - result_white)
- white_name = "candidate" if candidate_is_white else "baseline"
- black_name = "baseline" if candidate_is_white else "candidate"
+ # result_white is None for an aborted game (play_one_game's mover
+ # failed to produce a legal move for a non-resign reason) -- do
+ # NOT guess a winner (see abort_info's own comment at the
+ # return sites in play_one_game for why this matters: it's
+ # exactly the bug that silently corrupted ~47% of an earlier
+ # self-play corpus). Propagate None through candidate_result too
+ # so run()'s caller can skip it rather than score it.
+ candidate_result = (
+ None if result_white is None else
+ result_white if candidate_is_white else (1.0 - result_white)
+ )
+ white_name = args.candidate_name if candidate_is_white else args.baseline_name
+ black_name = args.baseline_name if candidate_is_white else args.candidate_name
game, _ = build_pgn_game(fen, moves, result_white, white_name, black_name,
move_evals=move_evals)
this_candidate_stats = white_stats if candidate_is_white else black_stats
this_head_stats = black_stats if candidate_is_white else white_stats
return (i, candidate_result, game, game_elapsed,
- this_candidate_stats, this_head_stats)
+ this_candidate_stats, this_head_stats, abort_info)
game_durations = []
job_iter = iter(jobs)
@@ -1057,14 +1232,39 @@ def main():
finished, in_flight = wait(in_flight, return_when=FIRST_COMPLETED)
for fut in finished:
(i, candidate_result, game, game_elapsed,
- this_candidate_stats, this_head_stats) = fut.result()
+ this_candidate_stats, this_head_stats, abort_info) = fut.result()
+ print(game, file=pgn_out, end="\n\n")
+ pgn_out.flush()
+
+ if candidate_result is None:
+ # Aborted game (play_one_game's mover failed to
+ # produce a legal move for a non-resign reason --
+ # see abort_info). Do NOT score it (no wins/
+ # losses/draws/candidate_points/SPRT update, no
+ # contribution to the `done` denominator used for
+ # score/ETA) -- this is exactly the "silently
+ # record a decisive result with no diagnostic
+ # trail" bug that corrupted ~47% of an earlier
+ # self-play corpus (docs/NNUE.md, 2026-10-17).
+ # The game is still written to pgn_out above
+ # (Result "*", which fit_k_and_label.py's
+ # result_to_white_score() already treats as
+ # "unknown, skip" for training) so it isn't lost
+ # for post-mortem debugging, just not scored.
+ aborted += 1
+ log.write(f"game {i}: ABORTED (not scored) "
+ f"abort_info={abort_info} "
+ f"elapsed={game_elapsed:.1f}s\n")
+ log.flush()
+ print(f" game {i}: ABORTED -- {abort_info}",
+ file=sys.stderr)
+ continue
+
done += 1
candidate_points += candidate_result
game_durations.append(game_elapsed)
candidate_stats.merge(this_candidate_stats)
head_stats.merge(this_head_stats)
- print(game, file=pgn_out, end="\n\n")
- pgn_out.flush()
if candidate_result == 1.0:
wins += 1
elif candidate_result == 0.0:
@@ -1131,7 +1331,7 @@ def main():
"(not waiting for in-flight games to finish naturally):",
file=sys.stderr)
print_final_summary(done, candidate_points, wins, draws,
- losses, candidate_stats, head_stats,
+ losses, aborted, candidate_stats, head_stats,
sprt, sprt_decision, args)
# os._exit, not sys.exit: skips the `with` block's normal
# ThreadPoolExecutor teardown (wait=True by default -- would
@@ -1143,17 +1343,21 @@ def main():
# the file objects' own __exit__.
os._exit(130)
- print_final_summary(done, candidate_points, wins, draws, losses,
+ print_final_summary(done, candidate_points, wins, draws, losses, aborted,
candidate_stats, head_stats, sprt, sprt_decision,
args)
-def print_final_summary(done, candidate_points, wins, draws, losses,
+def print_final_summary(done, candidate_points, wins, draws, losses, aborted,
candidate_stats, head_stats, sprt, sprt_decision,
args):
"""Shared by the normal end-of-match path and the ^C handler below --
same CANDIDATE_SCORE/CANDIDATE_STATS/HEAD_STATS block either way, just
- computed from whatever `done` games actually completed."""
+ computed from whatever `done` games actually completed. `aborted`
+ (not included in `done` or in any win/draw/loss/score arithmetic) is
+ the count of games where a mover failed to produce a legal move for
+ a non-resign reason -- see play_one_game's abort_info. Surfaced here
+ so an elevated count is visible instead of silently discarded."""
n_games = done
score = candidate_points / n_games if n_games else 0.5
elo, err = elo_diff(score, n_games)
@@ -1182,10 +1386,11 @@ def print_final_summary(done, candidate_points, wins, draws, losses,
f"cand={args.cand_time}s)")
else:
tc_str = f" TC=st{args.head_time}"
+ aborted_str = f" ABORTED={aborted}" if aborted else ""
print(f"CANDIDATE_SCORE={score:.4f} GAMES={n_games} "
f"WINS={wins} DRAWS={draws} LOSSES={losses} "
f"ELO={elo:+.1f}+/-{err:.1f} LOWER95={lower95:.4f}"
- f"{tc_str}{sprt_str}")
+ f"{tc_str}{sprt_str}{aborted_str}")
print(f"CANDIDATE_STATS: {candidate_stats.summary()}")
print(f"HEAD_STATS: {head_stats.summary()}")
diff --git a/src/eval_tune/nnue_vs_head.sh b/src/eval_tune/nnue_vs_head.sh
new file mode 100755
index 0000000..25a4c4f
--- /dev/null
+++ b/src/eval_tune/nnue_vs_head.sh
@@ -0,0 +1,219 @@
+#!/usr/bin/env bash
+#
+# Gated self-play match against a baseline binary. Historically this
+# always compared the live working-tree build against head_reference/
+# -- generalized (2026-09-14) to take both binary paths as args so two
+# arbitrary builds (e.g. a file-storm-only worktree build vs. the
+# current working tree, to isolate one change at a time when several
+# have landed together -- see king_safety_data/HANDOFF.md's session
+# notes on not measuring two eval changes in one match) can be compared
+# without hand-rolling a match_play.py invocation each time.
+#
+# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label]
+#
+# head_binary Baseline engine. Defaults to head_reference/typhoon
+# (the usual "is this better than last checkpoint"
+# question).
+# candidate_binary Candidate engine. Defaults to a fresh copy of the
+# live working-tree build (../typhoon) into
+# /tmp/typhoon/typhoon_match_play_candidate -- pass
+# an explicit path (already built) to skip that
+# copy/freeze step entirely, e.g. when comparing
+# two prebuilt worktree binaries.
+# label Distinguishes scratch dir/log/pgn-out so multiple
+# matches can run concurrently without colliding.
+# Defaults to "match".
+#
+# ...and any further arguments are passed straight through to
+# match_play.py, so a one-off flag no longer needs this file edited.
+#
+# Three env vars override the baked-in defaults (2026-10-03), so the
+# common knobs are reachable without touching the invocation below:
+#
+# TC_ARGS time control. Default: "--st 4"
+# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5"
+# GAMES game cap. Default: 10000
+#
+# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5"
+# cannot resolve the effect sizes this project's LMR work actually
+# produces. At the observed draw rate the standard error in Elo is about
+# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000
+# games (~53h here) -- and every LMR match run to date has instead
+# wandered around llr 0 for 2,000+ games and been abandoned, which is
+# exactly what a true effect sitting BETWEEN the two hypotheses looks
+# like. For a change you would only ship at >= 15 Elo, use
+# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours
+# instead of never.
+#
+# Examples:
+# test_vs_head.sh # old default behavior
+# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight
+#
+# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo:
+# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh
+#
+# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the
+# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in
+# # THIS engine, which bounds what any depth-buying search change
+# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity
+# # rather than testing a hypothesis, and ELO=+/- prints either way.
+# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo
+# # of precision already separates "a ply is worth 25" from "worth 65".
+# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \
+# test_vs_head.sh ../../head_reference/typhoon \
+# ../../head_reference/typhoon timeodds
+#
+# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/
+# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped
+# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS"
+# ahead of whatever args match_play.py's Engine class passes it --
+# that's the only change needed, since Engine spawns the given path
+# directly via Popen (no shell), so a shebang script in its place works
+# unmodified. HEAD is untouched, so it keeps running the classic HCE --
+# this is how you get an NNUE-candidate-vs-HCE-reference match without
+# patching match_play.py's Engine (which currently passes identical
+# engine_args to both sides):
+#
+# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh
+#
+# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by
+# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/
+# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to
+# also be set (the probe gate only does anything once NNUE eval itself
+# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the
+# same wrapper, enabling the lazy-eval accumulator probe gate at
+# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE
+# automatically, same convention as --nnueweights):
+#
+# NNUE_WEIGHTS=/path/to/net.nnue \
+# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \
+# test_vs_head.sh
+
+set -e
+
+# Only consume LEADING args that don't look like a flag as the three
+# positionals (head/candidate/label) -- 2026-10-19 fix: a bare
+# ./nnue_vs_head.sh --sd 10 used to land "--sd" in HEAD and "10" in
+# CANDIDATE_ARG (since the old code unconditionally took "$1"/"$2"/"$3"
+# regardless of content), the exact same footgun class the comment
+# below already describes once biting the --shift-then-test-"$2"
+# version of this script. A flag-looking first arg now means "no
+# positionals given at all, everything is pass-through" instead.
+POSITIONALS=()
+while [ $# -gt 0 ] && [ "${#POSITIONALS[@]}" -lt 3 ]; do
+ case "$1" in
+ -*) break ;;
+ *) POSITIONALS+=("$1"); shift ;;
+ esac
+done
+
+HEAD="${POSITIONALS[0]:-../../head_reference/typhoon}"
+CANDIDATE_ARG="${POSITIONALS[1]:-}"
+LABEL="${POSITIONALS[2]:-match}"
+# Note the `-` (not `:-`) for the three env overrides below: an
+# explicitly EMPTY value must be honored, not replaced by the default.
+# SPRT_ARGS="" is the documented way to run without a hypothesis test
+# (e.g. the time-odds calibration above), and `:-` would silently put
+# --sprt back.
+TC_ARGS="${TC_ARGS---st 4}"
+SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}"
+GAMES="${GAMES-10000}"
+
+# If the caller already passed --sd or --st through as an extra
+# argument (e.g. ./nnue_vs_head.sh --sd 10), don't ALSO inject the
+# baked-in TC_ARGS ("--st 4" by default) -- match_play.py's --sd/--st
+# are mutually exclusive, so the two would collide
+# ("error: argument --st: not allowed with argument --sd") even though
+# the caller only asked for one of them. Pass-through wins; TC_ARGS is
+# only the fallback when the caller didn't specify either.
+for arg in "$@"; do
+ case "$arg" in
+ --sd|--st) TC_ARGS="" ;;
+ esac
+done
+
+if [ -n "$CANDIDATE_ARG" ]; then
+ CANDIDATE="$CANDIDATE_ARG"
+else
+ cd ..
+ gmake clean
+ gmake release MP=1 PERF_COUNTERS=1
+ cd eval_tune
+ CANDIDATE=/tmp/typhoon/typhoon_match_play_candidate
+ cp -f ../typhoon_nnue ${CANDIDATE}
+ # typhoon_nnue's own TYPHOON_BIN/NNUE_WEIGHTS defaults are computed
+ # relative to ITS OWN location ("next to this script" -- see its
+ # header comment) -- correct when it's the real src/typhoon_nnue,
+ # but wrong once copied to /tmp/typhoon/ above: it would then look
+ # for the engine binary at /tmp/typhoon/typhoon and the weights
+ # file under /tmp/annotate_pgns/..., neither of which exist, and
+ # exit 1 immediately on every single spawn (2026-10-19: this is
+ # what produced hundreds of instant "BrokenPipeError" game aborts
+ # the first time this ran -- the copied wrapper was exiting before
+ # ever speaking the xboard protocol). match_play.py's Popen
+ # inherits this script's environment, so exporting the correct
+ # ABSOLUTE paths here -- computed from this script's own known-good
+ # location, not the relocated copy's -- fixes it without needing
+ # to touch typhoon_nnue itself. Only fill in NNUE_WEIGHTS if the
+ # caller didn't already set one explicitly.
+ export TYPHOON_BIN="$(cd .. && pwd)/typhoon"
+ : "${NNUE_WEIGHTS:=$(cd ../../annotate_pgns && pwd)/runs/sf98k/nnue_ckpt/nnue_weights.nnue}"
+ export NNUE_WEIGHTS
+fi
+
+if [ -n "${NNUE_WEIGHTS:-}" ]; then
+ NNUE_WRAPPER=/tmp/typhoon/typhoon_match_play_candidate_nnue
+ NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\""
+ if [ -n "${NNUE_PROBE_WEIGHTS:-}" ]; then
+ NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\""
+ fi
+ # NNUE_LAZY_MARGIN / NNUE_SUPERLAZY_MARGIN: 2026-10-16 looser-
+ # margin experiment (see docs/NNUE.md and command.c's "nnue
+ # lazymargin"/"nnue superlazyon"). NNUE_LAZY_MARGIN overrides every
+ # material bucket's lazy-probe margin with one flat cp value
+ # instead of the calibrated p98 table; NNUE_SUPERLAZY_MARGIN turns
+ # on the even-cheaper material-only pre-filter (off by default)
+ # with the given flat cp margin. Both require NNUE_WEIGHTS; the
+ # lazy one further requires NNUE_PROBE_WEIGHTS to do anything.
+ if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then
+ NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN"
+ fi
+ if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then
+ NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN"
+ fi
+ cat > "$NNUE_WRAPPER" <<WRAPPER
+#!/usr/bin/env bash
+exec "$CANDIDATE" $NNUE_FLAGS "\$@"
+WRAPPER
+ chmod +x "$NNUE_WRAPPER"
+ CANDIDATE="$NNUE_WRAPPER"
+fi
+
+# NNUE_CANDIDATE_NAME: PGN White/Black identity recorded for the
+# candidate engine, forwarded to match_play.py's --candidate-name
+# (2026-10-16, see that flag's own comment). Defaults to a name that
+# flags "don't trust this as training data" whenever NNUE is in play,
+# since self_play_games.pgn is reused later to train the NNUE net
+# itself and a differently-calibrated evaluator's scores shouldn't be
+# mixed in as labels without this being mechanically filterable.
+# Override explicitly if a given NNUE config's scores are actually
+# meant to be trusted.
+NAME_ARGS=""
+if [ -n "${NNUE_WEIGHTS:-}" ]; then
+ NNUE_CANDIDATE_NAME="${NNUE_CANDIDATE_NAME:-candidate_nnue}"
+ NAME_ARGS="--candidate-name $NNUE_CANDIDATE_NAME"
+fi
+
+python3 ./match_play.py \
+ ${HEAD} \
+ ${CANDIDATE} \
+ --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \
+ --games ${GAMES} \
+ --workers 18 \
+ --post \
+ ${TC_ARGS} \
+ ${SPRT_ARGS} \
+ ${NAME_ARGS} \
+ --scratch /usr/local/tmp/typhoon_match_${LABEL} \
+ --log ${LABEL}.log \
+ "$@"
diff --git a/src/eval_tune/relabel_selfplay_pgn.py b/src/eval_tune/relabel_selfplay_pgn.py
new file mode 100644
index 0000000..be14e22
--- /dev/null
+++ b/src/eval_tune/relabel_selfplay_pgn.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+One-off (but reusable) maintenance tool: retroactively rename the
+White/Black PGN header value for games matching a given Event + Date
+list, in place, without touching anything else in the file (moves,
+comments, other headers byte-for-byte unchanged).
+
+Why this exists, 2026-10-16: self_play_games.pgn is reused as NNUE
+training data (see docs/NNUE.md). match_play.py recorded every
+candidate engine as literally "candidate" regardless of what that
+candidate actually was, so the 2026-10-15/2026-10-16 batches (an NNUE
+candidate vs the HCE baseline, scores on the NNUE side compressed/
+uncalibrated -- see the engine-vs-engine eval discussion that same
+day) are indistinguishable from any other "candidate" batch by header
+alone. match_play.py now takes --candidate-name/--baseline-name to
+avoid this going forward; this script fixes the games written before
+that existed.
+
+Deliberately stream-edits the file line by line rather than round-
+tripping through python-chess's PGN writer -- re-serializing ~190k
+games risks reformatting move/comment text (whitespace, line wrapping)
+that downstream positional parsers (extract_positions.py) may depend
+on being byte-identical to how it was originally written. This script
+only ever rewrites a `[White "..."]`/`[Black "..."]` header line, and
+only when that game's own `[Date ...]`/`[Event ...]` headers already
+matched the target.
+
+Usage:
+ python3 relabel_selfplay_pgn.py <pgn-file> \\
+ --dates 2026-10-15 2026-10-16 \\
+ --event "typhoon self-play match" \\
+ --old-name candidate --new-name candidate_nnue_loose \\
+ [--dry-run]
+
+Writes to a sibling temp file and atomically replaces the original
+only after a full pass succeeds -- a failure partway through never
+leaves the original file half-rewritten.
+"""
+import argparse
+import os
+import re
+import sys
+
+
+HEADER_RE = re.compile(r'^\[(\w+)\s+"((?:[^"\\]|\\.)*)"\]\s*$')
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("pgn_file")
+ ap.add_argument("--dates", nargs="+", required=True,
+ help="Date header values (as PGN prints them, e.g. "
+ "2026-10-15) whose games are in scope")
+ ap.add_argument("--event", required=True,
+ help="Event header value a game must also match")
+ ap.add_argument("--old-name", required=True,
+ help="White/Black header value to rename")
+ ap.add_argument("--new-name", required=True,
+ help="replacement header value")
+ ap.add_argument("--dry-run", action="store_true",
+ help="report what would change, write nothing")
+ args = ap.parse_args()
+
+ target_dates = set(args.dates)
+ tmp_path = args.pgn_file + ".relabel_tmp"
+
+ cur_date = None
+ cur_event = None
+ in_scope = False # true once we know this game's headers match
+ games_seen = 0
+ games_touched = 0
+ renames = 0
+
+ with open(args.pgn_file, "r", encoding="utf-8", errors="surrogateescape") as fin, \
+ open(tmp_path, "w", encoding="utf-8", errors="surrogateescape") as fout:
+ for line in fin:
+ m = HEADER_RE.match(line)
+ if m:
+ tag, value = m.group(1), m.group(2)
+ if tag == "Event":
+ # New header block -> new game starting.
+ games_seen += 1
+ cur_event = value
+ cur_date = None
+ in_scope = False
+ elif tag == "Date":
+ cur_date = value
+ in_scope = (cur_event == args.event and cur_date in target_dates)
+ elif tag in ("White", "Black") and in_scope and value == args.old_name:
+ renames += 1
+ games_touched += 1 if tag == "White" else 0 # count once/game is enough
+ if not args.dry_run:
+ line = f'[{tag} "{args.new_name}"]\n'
+ fout.write(line)
+
+ if args.dry_run:
+ os.remove(tmp_path)
+ print(f"[dry run] {games_seen} games scanned, {renames} header "
+ f"values would be renamed ({args.old_name!r} -> "
+ f"{args.new_name!r}) for Event={args.event!r} Date in "
+ f"{sorted(target_dates)}")
+ return
+
+ os.replace(tmp_path, args.pgn_file)
+ print(f"{games_seen} games scanned, {renames} header values renamed "
+ f"({args.old_name!r} -> {args.new_name!r}) for Event={args.event!r} "
+ f"Date in {sorted(target_dates)}")
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/src/eval_tune/stockfish_training_corpus.sh b/src/eval_tune/stockfish_training_corpus.sh
new file mode 100755
index 0000000..75ce029
--- /dev/null
+++ b/src/eval_tune/stockfish_training_corpus.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+python3 ./match_play.py \
+ ../stockfish_xboard ../stockfish_xboard \
+ --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \
+ --pgn-out ../../pgn/stockfish_self_play_training_data.pgn \
+ --games 100000 \
+ --workers 100 \
+ --sd 10 \
+ --post \
+ --no-color-swap \
+ --scratch /usr/local/tmp/typhoon_match_sf_corpus \
+ --log /tmp/typhoon/sf_training_self_play.log
diff --git a/src/eval_tune/test_vs_head.sh b/src/eval_tune/test_vs_head.sh
index 71b2282..fa44d44 100755
--- a/src/eval_tune/test_vs_head.sh
+++ b/src/eval_tune/test_vs_head.sh
@@ -75,12 +75,41 @@
# engine_args to both sides):
#
# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh
+#
+# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by
+# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/
+# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to
+# also be set (the probe gate only does anything once NNUE eval itself
+# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the
+# same wrapper, enabling the lazy-eval accumulator probe gate at
+# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE
+# automatically, same convention as --nnueweights):
+#
+# NNUE_WEIGHTS=/path/to/net.nnue \
+# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \
+# test_vs_head.sh
set -e
-HEAD="${1:-../../head_reference/typhoon}"
-CANDIDATE_ARG="${2:-}"
-LABEL="${3:-match}"
+# Only consume LEADING args that don't look like a flag as the three
+# positionals (head/candidate/label) -- 2026-10-19 fix: a bare
+# ./test_vs_head.sh --sd 10 used to land "--sd" in HEAD and "10" in
+# CANDIDATE_ARG (since the old code unconditionally took "$1"/"$2"/"$3"
+# regardless of content), the exact same footgun class the comment
+# below already describes once biting the --shift-then-test-"$2"
+# version of this script. A flag-looking first arg now means "no
+# positionals given at all, everything is pass-through" instead.
+POSITIONALS=()
+while [ $# -gt 0 ] && [ "${#POSITIONALS[@]}" -lt 3 ]; do
+ case "$1" in
+ -*) break ;;
+ *) POSITIONALS+=("$1"); shift ;;
+ esac
+done
+
+HEAD="${POSITIONALS[0]:-../../head_reference/typhoon}"
+CANDIDATE_ARG="${POSITIONALS[1]:-}"
+LABEL="${POSITIONALS[2]:-match}"
# Note the `-` (not `:-`) for the three env overrides below: an
# explicitly EMPTY value must be honored, not replaced by the default.
# SPRT_ARGS="" is the documented way to run without a hypothesis test
@@ -90,18 +119,17 @@ TC_ARGS="${TC_ARGS---st 4}"
SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}"
GAMES="${GAMES-10000}"
-# Drop the (up to) three positional args so "$@" holds only extra
-# pass-through flags for match_play.py. This MUST come after the three
-# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above:
-# an earlier version of this shifted first and then tested "$2" to
-# decide whether to rebuild, which (a) made every no-extra-args run
-# take the rebuild branch and silently clobber ../typhoon with a
-# different make profile, and (b) made the first pass-through FLAG VALUE
-# land in "$2" and get used as the candidate BINARY PATH -- producing a
-# baffling "PermissionError: ... Permission denied: <scratch dir>" from
-# Popen trying to execute a directory.
-for _ in 1 2 3; do
- if [ $# -gt 0 ]; then shift; fi
+# If the caller already passed --sd or --st through as an extra
+# argument (e.g. ./test_vs_head.sh --sd 10), don't ALSO inject the
+# baked-in TC_ARGS ("--st 4" by default) -- match_play.py's --sd/--st
+# are mutually exclusive, so the two would collide
+# ("error: argument --st: not allowed with argument --sd") even though
+# the caller only asked for one of them. Pass-through wins; TC_ARGS is
+# only the fallback when the caller didn't specify either.
+for arg in "$@"; do
+ case "$arg" in
+ --sd|--st) TC_ARGS="" ;;
+ esac
done
if [ -n "$CANDIDATE_ARG" ]; then
@@ -117,14 +145,47 @@ fi
if [ -n "${NNUE_WEIGHTS:-}" ]; then
NNUE_WRAPPER=/tmp/typhoon/typhoon_match_play_candidate_nnue
+ NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\""
+ if [ -n "${NNUE_PROBE_WEIGHTS:-}" ]; then
+ NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\""
+ fi
+ # NNUE_LAZY_MARGIN / NNUE_SUPERLAZY_MARGIN: 2026-10-16 looser-
+ # margin experiment (see docs/NNUE.md and command.c's "nnue
+ # lazymargin"/"nnue superlazyon"). NNUE_LAZY_MARGIN overrides every
+ # material bucket's lazy-probe margin with one flat cp value
+ # instead of the calibrated p98 table; NNUE_SUPERLAZY_MARGIN turns
+ # on the even-cheaper material-only pre-filter (off by default)
+ # with the given flat cp margin. Both require NNUE_WEIGHTS; the
+ # lazy one further requires NNUE_PROBE_WEIGHTS to do anything.
+ if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then
+ NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN"
+ fi
+ if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then
+ NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN"
+ fi
cat > "$NNUE_WRAPPER" <<WRAPPER
#!/usr/bin/env bash
-exec "$CANDIDATE" --nnueweights "$NNUE_WEIGHTS" "\$@"
+exec "$CANDIDATE" $NNUE_FLAGS "\$@"
WRAPPER
chmod +x "$NNUE_WRAPPER"
CANDIDATE="$NNUE_WRAPPER"
fi
+# NNUE_CANDIDATE_NAME: PGN White/Black identity recorded for the
+# candidate engine, forwarded to match_play.py's --candidate-name
+# (2026-10-16, see that flag's own comment). Defaults to a name that
+# flags "don't trust this as training data" whenever NNUE is in play,
+# since self_play_games.pgn is reused later to train the NNUE net
+# itself and a differently-calibrated evaluator's scores shouldn't be
+# mixed in as labels without this being mechanically filterable.
+# Override explicitly if a given NNUE config's scores are actually
+# meant to be trusted.
+NAME_ARGS=""
+if [ -n "${NNUE_WEIGHTS:-}" ]; then
+ NNUE_CANDIDATE_NAME="${NNUE_CANDIDATE_NAME:-candidate_nnue}"
+ NAME_ARGS="--candidate-name $NNUE_CANDIDATE_NAME"
+fi
+
python3 ./match_play.py \
${HEAD} \
${CANDIDATE} \
@@ -134,6 +195,7 @@ python3 ./match_play.py \
--post \
${TC_ARGS} \
${SPRT_ARGS} \
+ ${NAME_ARGS} \
--scratch /usr/local/tmp/typhoon_match_${LABEL} \
--log ${LABEL}.log \
"$@"
diff --git a/src/eval_tune/typhoon_hce_training_corpus.sh b/src/eval_tune/typhoon_hce_training_corpus.sh
new file mode 100755
index 0000000..1fc16d8
--- /dev/null
+++ b/src/eval_tune/typhoon_hce_training_corpus.sh
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+#
+# Gated self-play match against a baseline binary. Historically this
+# always compared the live working-tree build against head_reference/
+# -- generalized (2026-09-14) to take both binary paths as args so two
+# arbitrary builds (e.g. a file-storm-only worktree build vs. the
+# current working tree, to isolate one change at a time when several
+# have landed together -- see king_safety_data/HANDOFF.md's session
+# notes on not measuring two eval changes in one match) can be compared
+# without hand-rolling a match_play.py invocation each time.
+#
+# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label]
+#
+# head_binary Baseline engine. Defaults to a fresh copy of the
+# SAME just-built working-tree binary the candidate
+# uses (/tmp/typhoon/typhoon_match_play_head) --
+# this is self-play training-data generation, not a
+# "better than last checkpoint" gate, so baseline
+# and candidate must share every runtime fix/option
+# (e.g. the PostThrottle >= fix); a stale
+# head_reference/typhoon silently lacks those and
+# produces training data with gaps that look like
+# search bugs but are actually just an old binary.
+# Pass an explicit path (already built) -- together
+# with candidate_binary -- to skip the rebuild/copy
+# step entirely, e.g. when comparing two prebuilt
+# worktree binaries.
+# candidate_binary Candidate engine. Defaults to a fresh copy of the
+# live working-tree build (../typhoon) into
+# /tmp/typhoon/typhoon_match_play_candidate. Must be
+# given together with head_binary -- either both
+# positionals are prebuilt paths, or neither is and
+# both get rebuilt copies.
+# label Distinguishes scratch dir/log/pgn-out so multiple
+# matches can run concurrently without colliding.
+# Defaults to "match".
+#
+# ...and any further arguments are passed straight through to
+# match_play.py, so a one-off flag no longer needs this file edited.
+#
+# Three env vars override the baked-in defaults (2026-10-03), so the
+# common knobs are reachable without touching the invocation below:
+#
+# TC_ARGS time control. Default: "--st 4"
+# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5"
+# GAMES game cap. Default: 10000
+#
+# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5"
+# cannot resolve the effect sizes this project's LMR work actually
+# produces. At the observed draw rate the standard error in Elo is about
+# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000
+# games (~53h here) -- and every LMR match run to date has instead
+# wandered around llr 0 for 2,000+ games and been abandoned, which is
+# exactly what a true effect sitting BETWEEN the two hypotheses looks
+# like. For a change you would only ship at >= 15 Elo, use
+# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours
+# instead of never.
+#
+# Examples:
+# test_vs_head.sh # old default behavior
+# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight
+#
+# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo:
+# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh
+#
+# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the
+# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in
+# # THIS engine, which bounds what any depth-buying search change
+# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity
+# # rather than testing a hypothesis, and ELO=+/- prints either way.
+# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo
+# # of precision already separates "a ply is worth 25" from "worth 65".
+# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \
+# test_vs_head.sh ../../head_reference/typhoon \
+# ../../head_reference/typhoon timeodds
+#
+# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/
+# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped
+# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS"
+# ahead of whatever args match_play.py's Engine class passes it --
+# that's the only change needed, since Engine spawns the given path
+# directly via Popen (no shell), so a shebang script in its place works
+# unmodified. HEAD is untouched, so it keeps running the classic HCE --
+# this is how you get an NNUE-candidate-vs-HCE-reference match without
+# patching match_play.py's Engine (which currently passes identical
+# engine_args to both sides):
+#
+# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh
+#
+# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by
+# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/
+# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to
+# also be set (the probe gate only does anything once NNUE eval itself
+# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the
+# same wrapper, enabling the lazy-eval accumulator probe gate at
+# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE
+# automatically, same convention as --nnueweights):
+#
+# NNUE_WEIGHTS=/path/to/net.nnue \
+# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \
+# test_vs_head.sh
+
+set -e
+
+HEAD_ARG="${1:-}"
+CANDIDATE_ARG="${2:-}"
+LABEL="${3:-match}"
+# Note the `-` (not `:-`) for the three env overrides below: an
+# explicitly EMPTY value must be honored, not replaced by the default.
+# SPRT_ARGS="" is the documented way to run without a hypothesis test
+# (e.g. the time-odds calibration above), and `:-` would silently put
+# --sprt back.
+#TC_ARGS="${TC_ARGS---st 4}"
+#SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}"
+GAMES="${GAMES-10000}"
+
+# Drop the (up to) three positional args so "$@" holds only extra
+# pass-through flags for match_play.py. This MUST come after the three
+# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above:
+# an earlier version of this shifted first and then tested "$2" to
+# decide whether to rebuild, which (a) made every no-extra-args run
+# take the rebuild branch and silently clobber ../typhoon with a
+# different make profile, and (b) made the first pass-through FLAG VALUE
+# land in "$2" and get used as the candidate BINARY PATH -- producing a
+# baffling "PermissionError: ... Permission denied: <scratch dir>" from
+# Popen trying to execute a directory.
+for _ in 1 2 3; do
+ if [ $# -gt 0 ]; then shift; fi
+done
+
+if [ -n "$HEAD_ARG" ] && [ -n "$CANDIDATE_ARG" ]; then
+ HEAD="$HEAD_ARG"
+ CANDIDATE="$CANDIDATE_ARG"
+else
+ cd ..
+ gmake clean
+ gmake release MP=1 PERF_COUNTERS=1
+ cd eval_tune
+ # Both sides get a fresh copy of the SAME just-built ../typhoon --
+ # unlike head_reference/typhoon (a stale checked-in binary), this
+ # guarantees baseline and candidate share every runtime fix/option
+ # (e.g. the PostThrottle >= fix) so a missing comment in training
+ # data can't be blamed on baseline running old code. Pass both
+ # HEAD_ARG and CANDIDATE_ARG explicitly to compare two prebuilt
+ # binaries instead.
+ HEAD="${HEAD_ARG:-/tmp/typhoon/typhoon_match_play_head}"
+ CANDIDATE="${CANDIDATE_ARG:-/tmp/typhoon/typhoon_match_play_candidate}"
+ cp -f ../typhoon ${HEAD}
+ cp -f ../typhoon ${CANDIDATE}
+fi
+
+# --no-color-swap: head and candidate are copies of the SAME binary
+# here (this is corpus generation, not an Elo gate), so match_play.py's
+# default color-swap-per-opening would just play every opening TWICE
+# and produce a byte-for-byte duplicate game the second time (confirmed
+# 2026-10-16 against nnue_self_play_training_data.pgn) -- pure wasted
+# CPU for a training corpus that wants variety, not duplication. See
+# --no-color-swap's own help text in match_play.py for the full
+# reasoning.
+python3 ./match_play.py \
+ ${HEAD} \
+ ${CANDIDATE} \
+ --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \
+ --pgn-out ../../pgn/nnue_self_play_training_data.pgn \
+ --games ${GAMES} \
+ --workers 26 \
+ --sd 10 \
+ --post \
+ --no-color-swap \
+ ${NAME_ARGS} \
+ --scratch /usr/local/tmp/typhoon_match_${LABEL} \
+ --log /tmp/typhoon/nnue_training_self_play.log \
+ "$@"
diff --git a/src/eval_tune/typhoon_nnue_training_corpus.sh b/src/eval_tune/typhoon_nnue_training_corpus.sh
new file mode 100755
index 0000000..b1a00d4
--- /dev/null
+++ b/src/eval_tune/typhoon_nnue_training_corpus.sh
@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+#
+# Gated self-play match against a baseline binary. Historically this
+# always compared the live working-tree build against head_reference/
+# -- generalized (2026-09-14) to take both binary paths as args so two
+# arbitrary builds (e.g. a file-storm-only worktree build vs. the
+# current working tree, to isolate one change at a time when several
+# have landed together -- see king_safety_data/HANDOFF.md's session
+# notes on not measuring two eval changes in one match) can be compared
+# without hand-rolling a match_play.py invocation each time.
+#
+# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label]
+#
+# head_binary Baseline engine. Defaults to a fresh copy of the
+# SAME just-built working-tree binary the candidate
+# uses (/tmp/typhoon/typhoon_match_play_head) --
+# this is self-play training-data generation, not a
+# "better than last checkpoint" gate, so baseline
+# and candidate must share every runtime fix/option
+# (e.g. the PostThrottle >= fix); a stale
+# head_reference/typhoon silently lacks those and
+# produces training data with gaps that look like
+# search bugs but are actually just an old binary.
+# Pass an explicit path (already built) -- together
+# with candidate_binary -- to skip the rebuild/copy
+# step entirely, e.g. when comparing two prebuilt
+# worktree binaries.
+# candidate_binary Candidate engine. Defaults to a fresh copy of the
+# live working-tree build (../typhoon) into
+# /tmp/typhoon/typhoon_match_play_candidate. Must be
+# given together with head_binary -- either both
+# positionals are prebuilt paths, or neither is and
+# both get rebuilt copies.
+# label Distinguishes scratch dir/log/pgn-out so multiple
+# matches can run concurrently without colliding.
+# Defaults to "match".
+#
+# ...and any further arguments are passed straight through to
+# match_play.py, so a one-off flag no longer needs this file edited.
+#
+# Three env vars override the baked-in defaults (2026-10-03), so the
+# common knobs are reachable without touching the invocation below:
+#
+# TC_ARGS time control. Default: "--st 4"
+# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5"
+# GAMES game cap. Default: 10000
+#
+# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5"
+# cannot resolve the effect sizes this project's LMR work actually
+# produces. At the observed draw rate the standard error in Elo is about
+# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000
+# games (~53h here) -- and every LMR match run to date has instead
+# wandered around llr 0 for 2,000+ games and been abandoned, which is
+# exactly what a true effect sitting BETWEEN the two hypotheses looks
+# like. For a change you would only ship at >= 15 Elo, use
+# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours
+# instead of never.
+#
+# Examples:
+# test_vs_head.sh # old default behavior
+# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight
+#
+# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo:
+# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh
+#
+# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the
+# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in
+# # THIS engine, which bounds what any depth-buying search change
+# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity
+# # rather than testing a hypothesis, and ELO=+/- prints either way.
+# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo
+# # of precision already separates "a ply is worth 25" from "worth 65".
+# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \
+# test_vs_head.sh ../../head_reference/typhoon \
+# ../../head_reference/typhoon timeodds
+#
+# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/
+# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped
+# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS"
+# ahead of whatever args match_play.py's Engine class passes it --
+# that's the only change needed, since Engine spawns the given path
+# directly via Popen (no shell), so a shebang script in its place works
+# unmodified. HEAD is untouched, so it keeps running the classic HCE --
+# this is how you get an NNUE-candidate-vs-HCE-reference match without
+# patching match_play.py's Engine (which currently passes identical
+# engine_args to both sides):
+#
+# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh
+#
+# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by
+# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/
+# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to
+# also be set (the probe gate only does anything once NNUE eval itself
+# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the
+# same wrapper, enabling the lazy-eval accumulator probe gate at
+# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE
+# automatically, same convention as --nnueweights):
+#
+# NNUE_WEIGHTS=/path/to/net.nnue \
+# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \
+# test_vs_head.sh
+
+set -e
+
+HEAD_ARG="${1:-}"
+CANDIDATE_ARG="${2:-}"
+LABEL="${3:-match}"
+# Note the `-` (not `:-`) for the three env overrides below: an
+# explicitly EMPTY value must be honored, not replaced by the default.
+# SPRT_ARGS="" is the documented way to run without a hypothesis test
+# (e.g. the time-odds calibration above), and `:-` would silently put
+# --sprt back.
+#TC_ARGS="${TC_ARGS---st 4}"
+#SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}"
+GAMES="${GAMES-10000}"
+
+# Drop the (up to) three positional args so "$@" holds only extra
+# pass-through flags for match_play.py. This MUST come after the three
+# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above:
+# an earlier version of this shifted first and then tested "$2" to
+# decide whether to rebuild, which (a) made every no-extra-args run
+# take the rebuild branch and silently clobber ../typhoon with a
+# different make profile, and (b) made the first pass-through FLAG VALUE
+# land in "$2" and get used as the candidate BINARY PATH -- producing a
+# baffling "PermissionError: ... Permission denied: <scratch dir>" from
+# Popen trying to execute a directory.
+for _ in 1 2 3; do
+ if [ $# -gt 0 ]; then shift; fi
+done
+
+if [ -n "$HEAD_ARG" ] && [ -n "$CANDIDATE_ARG" ]; then
+ HEAD="$HEAD_ARG"
+ CANDIDATE="$CANDIDATE_ARG"
+else
+ cd ..
+ gmake clean
+ gmake release MP=1 PERF_COUNTERS=1
+ cd eval_tune
+ # Both sides get a fresh copy of the SAME just-built ../typhoon --
+ # unlike head_reference/typhoon (a stale checked-in binary), this
+ # guarantees baseline and candidate share every runtime fix/option
+ # (e.g. the PostThrottle >= fix) so a missing comment in training
+ # data can't be blamed on baseline running old code. Pass both
+ # HEAD_ARG and CANDIDATE_ARG explicitly to compare two prebuilt
+ # binaries instead.
+ HEAD="${HEAD_ARG:-/tmp/typhoon/typhoon_match_play_nnue1}"
+ CANDIDATE="${CANDIDATE_ARG:-/tmp/typhoon/typhoon_match_play_nnue2}"
+ cp -f ../typhoon ${HEAD}
+ cp -f ../typhoon ${CANDIDATE}
+fi
+
+# --no-color-swap: head and candidate are copies of the SAME binary
+# here (this is corpus generation, not an Elo gate), so match_play.py's
+# default color-swap-per-opening would just play every opening TWICE
+# and produce a byte-for-byte duplicate game the second time (confirmed
+# 2026-10-16 against nnue_self_play_training_data.pgn) -- pure wasted
+# CPU for a training corpus that wants variety, not duplication. See
+# --no-color-swap's own help text in match_play.py for the full
+# reasoning.
+NNUE_WEIGHTS="${NNUE_WEIGHTS:-$HOME/typhoon/annotate_pgns/runs/sf98k/nnue_ckpt/nnue_weights.nnue}"
+if [ ! -f "$NNUE_WEIGHTS" ]; then
+ echo "typhoon_nnue: NNUE_WEIGHTS file not found: $NNUE_WEIGHTS" >&2
+ exit 1
+fi
+
+NNUE_PROBE_WEIGHTS="${NNUE_PROBE_WEIGHTS-}"
+
+# Both HEAD and CANDIDATE are the same just-built binary (this is NNUE
+# self-play training-data generation, not an HCE-vs-NNUE Elo gate), so
+# the NNUE flags get wrapped into exec shims for BOTH sides -- a bare
+# string-concatenation of path+flags doesn't work here: match_play.py's
+# head_engine/candidate_engine are resolved via Path(...).resolve() as a
+# single filesystem path, so passing "$HEAD $ARGS" as one argument would
+# just produce a nonexistent path containing literal spaces. See
+# test_vs_head.sh's NNUE_WRAPPER for the same pattern, used there to wrap
+# only the candidate (HCE head vs NNUE candidate); here both sides need
+# the net since both sides generate training data as NNUE.
+NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\""
+if [ -n "$NNUE_PROBE_WEIGHTS" ] && [ -f "$NNUE_PROBE_WEIGHTS" ]; then
+ NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\""
+ if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then
+ NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN"
+ fi
+fi
+if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then
+ NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN"
+fi
+
+HEAD_WRAPPER=/tmp/typhoon/typhoon_match_play_nnue1_wrapped
+cat > "$HEAD_WRAPPER" <<WRAPPER
+#!/usr/bin/env bash
+exec "$HEAD" $NNUE_FLAGS "\$@"
+WRAPPER
+chmod +x "$HEAD_WRAPPER"
+HEAD="$HEAD_WRAPPER"
+
+CANDIDATE_WRAPPER=/tmp/typhoon/typhoon_match_play_nnue2_wrapped
+cat > "$CANDIDATE_WRAPPER" <<WRAPPER
+#!/usr/bin/env bash
+exec "$CANDIDATE" $NNUE_FLAGS "\$@"
+WRAPPER
+chmod +x "$CANDIDATE_WRAPPER"
+CANDIDATE="$CANDIDATE_WRAPPER"
+
+# See test_vs_head.sh's NNUE_CANDIDATE_NAME comment: flags the PGN
+# White/Black identity so NNUE-evaluated self-play games are
+# mechanically filterable out of training data that isn't meant to
+# mix differently-calibrated evaluator scores.
+NAME_ARGS="--candidate-name ${NNUE_CANDIDATE_NAME:-candidate_nnue} --baseline-name ${NNUE_HEAD_NAME:-head_nnue}"
+
+python3 ./match_play.py \
+ "${HEAD}" \
+ "${CANDIDATE}" \
+ --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \
+ --pgn-out ../../pgn/nnue_self_play_training_data.pgn \
+ --games ${GAMES} \
+ --workers 26 \
+ --sd 10 \
+ --post \
+ --no-color-swap \
+ ${NAME_ARGS} \
+ --scratch /usr/local/tmp/typhoon_match_${LABEL} \
+ --log /tmp/typhoon/nnue_training_self_play.log \
+ "$@"
diff --git a/src/main.c b/src/main.c
index 90301af..f822b7e 100644
--- a/src/main.c
+++ b/src/main.c
@@ -305,6 +305,7 @@ Return value:
g_Options.uNumProcessors = 1;
g_Options.fStatusLine = TRUE;
g_Options.iResignThreshold = 0;
+ g_Options.dPostThrottleSec = 0.35;
g_Options.u64MaxNodeCount = 0ULL;
i = 1;
@@ -424,6 +425,53 @@ Return value:
}
if (fConsumedNextArg) i++;
}
+ else if ((!STRCMPI(pszArg, "--nnueprobeweights")) && fHaveVal)
+ {
+ // Same reasoning as --nnueweights above -- loads and
+ // enables the lazy-eval accumulator probe gate at startup
+ // for scripted match-play, where there's no interactive
+ // "nnue probeload"/"nnue lazyprobeon" command pair
+ // available. Does NOT imply --nnueweights -- pass both to
+ // get the full NNUE+lazy-gate configuration; this alone
+ // without NNUE eval enabled would load the probe but never
+ // reach the code path that consults it.
+ if (!NnueProbeLoadWeights(pszVal))
+ {
+ UtilPanic(INITIALIZATION_FAILURE,
+ (void *)"Bad NNUE probe weights file",
+ (void *)&(pszVal),
+ (void *)"probeload",
+ NULL,
+ __FILE__, __LINE__);
+ }
+ else
+ {
+ g_fUseNnueLazyProbe = TRUE;
+ Trace("Loaded NNUE probe weights \"%s\", lazy probe gate enabled.\n",
+ pszVal);
+ }
+ if (fConsumedNextArg) i++;
+ }
+ else if ((!STRCMPI(pszArg, "--nnuelazymargin")) && fHaveVal)
+ {
+ // Scripted-match-play equivalent of "nnue lazymargin <cp>"
+ // -- flat override for every material bucket's lazy-probe
+ // margin, see nnueeval.c/eval.c. 2026-10-16 looser-margin
+ // experiment.
+ g_iNnueLazyMarginOverride = atoi(pszVal);
+ Trace("NNUE lazy margin override set to %d cp.\n",
+ g_iNnueLazyMarginOverride);
+ if (fConsumedNextArg) i++;
+ }
+ else if ((!STRCMPI(pszArg, "--nnuesuperlazymargin")) && fHaveVal)
+ {
+ // Scripted equivalent of "nnue superlazyon <cp>".
+ g_iNnueSuperLazyMargin = (SCORE)atoi(pszVal);
+ g_fUseNnueSuperLazy = TRUE;
+ Trace("NNUE super-lazy pre-filter enabled, margin=%d cp.\n",
+ g_iNnueSuperLazyMargin);
+ if (fConsumedNextArg) i++;
+ }
else if ((!STRCMPI(pszArg, "--dnafile")) && fHaveVal)
{
if (!ReadEvalDNA(pszVal))
diff --git a/src/nnueaccum.c b/src/nnueaccum.c
index 8f01692..7a3010f 100644
--- a/src/nnueaccum.c
+++ b/src/nnueaccum.c
@@ -34,16 +34,12 @@ Author:
#include "chess.h"
-// Placeholder embedding table -- see chess.h's extern declaration and
-// its own comment. Zero-initialized (C's static/global default), so
-// every accumulator is currently all-zero regardless of position;
-// real weight loading (reading annotate_pgns/nnue_train.py's trained
-// checkpoint) is separate, not-yet-built follow-up work. This is
-// sufficient to prove incremental-vs-recompute-from-scratch
-// correctness, which only depends on both paths reading the SAME
-// table consistently, not on its values being meaningful yet.
-float g_rgfNnueWeight[NNUE_DIMENSIONS][NNUE_ACCUMULATOR_SIZE];
-float g_rgfNnueBias[NNUE_ACCUMULATOR_SIZE];
+// Quantized embedding table -- see chess.h's extern declaration and
+// its own comment. Zero-initialized (C's static/global default) until
+// NnueLoadWeights reads a real exported checkpoint.
+SHORT g_rgfNnueWeight[NNUE_DIMENSIONS][NNUE_ACCUMULATOR_SIZE];
+SHORT g_rgfNnueBias[NNUE_ACCUMULATOR_SIZE];
+float g_fNnueFtScale = 1.0f;
void
NnueAccumAddFeature(NNUE_ACCUMULATOR *pAccum, ULONG uPerspective,
@@ -57,12 +53,12 @@ Routine description:
**/
{
int idx = NnueFeatureIndex(uPerspective, cKingOwn, cPieceSquare, pPiece);
- const float *pRow = g_rgfNnueWeight[idx];
+ const SHORT *pRow = g_rgfNnueWeight[idx];
ULONG i;
for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
{
- pAccum->rgfValue[i] += pRow[i];
+ pAccum->rgiValue[i] += pRow[i];
}
}
@@ -76,18 +72,18 @@ Routine description:
Subtracts one feature's weight row from *pAccum (in place). Hot
path -- the exact inverse of NnueAccumAddFeature, so that applying
- Add then Remove (or Remove then Add) for the same arguments is a
- no-op up to float rounding.
+ Add then Remove (or Remove then Add) for the same arguments is an
+ exact no-op (integer arithmetic, no float rounding to worry about).
**/
{
int idx = NnueFeatureIndex(uPerspective, cKingOwn, cPieceSquare, pPiece);
- const float *pRow = g_rgfNnueWeight[idx];
+ const SHORT *pRow = g_rgfNnueWeight[idx];
ULONG i;
for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
{
- pAccum->rgfValue[i] -= pRow[i];
+ pAccum->rgiValue[i] -= pRow[i];
}
}
@@ -125,7 +121,7 @@ Parameters:
for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
{
- pAccum->rgfValue[i] = g_rgfNnueBias[i];
+ pAccum->rgiValue[i] = g_rgfNnueBias[i];
}
// 0x88 COORs aren't a simple monotonic range (A1=0x70 > H8=0x07),
diff --git a/src/nnueeval.c b/src/nnueeval.c
index 6ee57d5..7caadef 100644
--- a/src/nnueeval.c
+++ b/src/nnueeval.c
@@ -28,13 +28,23 @@ Author:
**/
#include "chess.h"
+#include <float.h>
-float g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE];
+// g_rgfNnueWeight/g_rgfNnueBias (SHORT, quantized) and g_fNnueFtScale
+// are defined in nnueaccum.c. The FC head below is quantized to CHAR
+// (int8) per docs/NNUE.md's "Quantization plan" step 2 -- biases stay
+// float32 (tiny, added post-dequant). g_rgfNnueActScale/WeightScale
+// are read from the export file's header; defaults of 1.0f here only
+// matter before any real weights are loaded (the all-zero placeholder
+// table makes every activation 0 regardless of scale).
+CHAR g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE];
float g_rgfNnueFc1Bias[NNUE_L2];
-float g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2];
+CHAR g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2];
float g_rgfNnueFc2Bias[NNUE_L3];
-float g_rgfNnueFc3Weight[NNUE_L3];
+CHAR g_rgfNnueFc3Weight[NNUE_L3];
float g_rgfNnueFc3Bias;
+float g_rgfNnueActScale[3] = { 1.0f, 1.0f, 1.0f };
+float g_rgfNnueWeightScale[3] = { 1.0f, 1.0f, 1.0f };
FLAG g_fNnueWeightsLoaded = FALSE;
FLAG g_fUseNnueEval = FALSE;
@@ -76,10 +86,12 @@ Return value:
}
if ((1 != fread(szMagic, 4, 1, fp)) ||
- (0 != memcmp(szMagic, "NNU1", 4)))
+ (0 != memcmp(szMagic, "NNU3", 4)))
{
Trace("NnueLoadWeights: %s is not a valid NNUE weights file "
- "(bad magic)\n", szFilename);
+ "(bad magic -- expected \"NNU3\"; an older \"NNU1\"/"
+ "\"NNU2\" file predates the FC head's int8 quantization "
+ "and is no longer readable)\n", szFilename);
fclose(fp);
return FALSE;
}
@@ -104,9 +116,24 @@ Return value:
return FALSE;
}
-#define READ_OR_FAIL(ptr, count) \
+ if (1 != fread(&g_fNnueFtScale, sizeof(g_fNnueFtScale), 1, fp))
+ {
+ Trace("NnueLoadWeights: %s truncated (ft_scale)\n", szFilename);
+ fclose(fp);
+ return FALSE;
+ }
+ if ((1 != fread(g_rgfNnueActScale, sizeof(g_rgfNnueActScale), 1, fp)) ||
+ (1 != fread(g_rgfNnueWeightScale, sizeof(g_rgfNnueWeightScale), 1, fp)))
+ {
+ Trace("NnueLoadWeights: %s truncated (act/weight scales)\n",
+ szFilename);
+ fclose(fp);
+ return FALSE;
+ }
+
+#define READ_OR_FAIL(ptr, elsize, count) \
do { \
- if ((size_t)(count) != fread((ptr), sizeof(float), (count), fp)) \
+ if ((size_t)(count) != fread((ptr), (elsize), (count), fp)) \
{ \
Trace("NnueLoadWeights: %s truncated\n", szFilename); \
fclose(fp); \
@@ -114,14 +141,16 @@ Return value:
} \
} while (0)
- READ_OR_FAIL(g_rgfNnueWeight, (size_t)NNUE_DIMENSIONS * NNUE_ACCUMULATOR_SIZE);
- READ_OR_FAIL(g_rgfNnueBias, NNUE_ACCUMULATOR_SIZE);
- READ_OR_FAIL(g_rgfNnueFc1Weight, (size_t)NNUE_L2 * 2 * NNUE_ACCUMULATOR_SIZE);
- READ_OR_FAIL(g_rgfNnueFc1Bias, NNUE_L2);
- READ_OR_FAIL(g_rgfNnueFc2Weight, (size_t)NNUE_L3 * NNUE_L2);
- READ_OR_FAIL(g_rgfNnueFc2Bias, NNUE_L3);
- READ_OR_FAIL(g_rgfNnueFc3Weight, NNUE_L3);
- READ_OR_FAIL(&g_rgfNnueFc3Bias, 1);
+ READ_OR_FAIL(g_rgfNnueWeight, sizeof(SHORT),
+ (size_t)NNUE_DIMENSIONS * NNUE_ACCUMULATOR_SIZE);
+ READ_OR_FAIL(g_rgfNnueBias, sizeof(SHORT), NNUE_ACCUMULATOR_SIZE);
+ READ_OR_FAIL(g_rgfNnueFc1Weight, sizeof(CHAR),
+ (size_t)NNUE_L2 * 2 * NNUE_ACCUMULATOR_SIZE);
+ READ_OR_FAIL(g_rgfNnueFc1Bias, sizeof(float), NNUE_L2);
+ READ_OR_FAIL(g_rgfNnueFc2Weight, sizeof(CHAR), (size_t)NNUE_L3 * NNUE_L2);
+ READ_OR_FAIL(g_rgfNnueFc2Bias, sizeof(float), NNUE_L3);
+ READ_OR_FAIL(g_rgfNnueFc3Weight, sizeof(CHAR), NNUE_L3);
+ READ_OR_FAIL(&g_rgfNnueFc3Bias, sizeof(float), 1);
#undef READ_OR_FAIL
@@ -163,48 +192,367 @@ Return value:
POSITION *pos = &ctx->sPosition;
ULONG uStm = pos->uToMove;
ULONG uOther = FLIP(uStm);
- float rgfConcat[2 * NNUE_ACCUMULATOR_SIZE];
+ UCHAR rgbAct1[2 * NNUE_ACCUMULATOR_SIZE]; // quantized fc1 input
+ UCHAR rgbAct2[NNUE_L2]; // quantized fc2 input
+ UCHAR rgbAct3[NNUE_L3]; // quantized fc3 input
float rgfL2[NNUE_L2];
float rgfL3[NNUE_L3];
float fOut;
ULONG i, j;
+ // Stage 0: dequantize the INT accumulator (scaled by
+ // g_fNnueFtScale, see chess.h's NNUE_ACCUMULATOR comment) back to
+ // float, apply ReLU, then requantize to UCHAR at AS1
+ // (g_rgfNnueActScale[0]) -- this is "fc1's input" in the Python
+ // reference. Own-perspective-first concatenation matches the
+ // side-to-move-relative output convention.
for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
{
- float v = ctx->sNnueAccum[uStm].rgfValue[i];
- rgfConcat[i] = (v > 0.0f) ? v : 0.0f;
+ float v = (float)ctx->sNnueAccum[uStm].rgiValue[i] / g_fNnueFtScale;
+ v = (v > 0.0f) ? v : 0.0f;
+ rgbAct1[i] = (UCHAR)MAX(0, MIN(255, (int)(v * g_rgfNnueActScale[0] + 0.5f)));
}
for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
{
- float v = ctx->sNnueAccum[uOther].rgfValue[i];
- rgfConcat[NNUE_ACCUMULATOR_SIZE + i] = (v > 0.0f) ? v : 0.0f;
+ float v = (float)ctx->sNnueAccum[uOther].rgiValue[i] / g_fNnueFtScale;
+ v = (v > 0.0f) ? v : 0.0f;
+ rgbAct1[NNUE_ACCUMULATOR_SIZE + i] =
+ (UCHAR)MAX(0, MIN(255, (int)(v * g_rgfNnueActScale[0] + 0.5f)));
}
+ // Stage 1: fc1, int8 dot product (UCHAR activation x CHAR weight)
+ // accumulated in INT (32-bit) -- up to 512 terms, each at most
+ // 255*127, nowhere near INT overflow. Dequantize by dividing out
+ // both scales, add the (unquantized) bias, ReLU, requantize at
+ // AS2 for fc2's input.
for (j = 0; j < NNUE_L2; j++)
{
- float s = g_rgfNnueFc1Bias[j];
+ INT iSum = 0;
for (i = 0; i < 2 * NNUE_ACCUMULATOR_SIZE; i++)
{
- s += g_rgfNnueFc1Weight[j][i] * rgfConcat[i];
+ iSum += (INT)rgbAct1[i] * (INT)g_rgfNnueFc1Weight[j][i];
}
- rgfL2[j] = (s > 0.0f) ? s : 0.0f;
+ float s = ((float)iSum / (g_rgfNnueActScale[0] * g_rgfNnueWeightScale[0])) +
+ g_rgfNnueFc1Bias[j];
+ s = (s > 0.0f) ? s : 0.0f;
+ rgfL2[j] = s;
+ rgbAct2[j] = (UCHAR)MAX(0, MIN(255, (int)(s * g_rgfNnueActScale[1] + 0.5f)));
}
+ // Stage 2: fc2, same pattern.
for (j = 0; j < NNUE_L3; j++)
{
- float s = g_rgfNnueFc2Bias[j];
+ INT iSum = 0;
for (i = 0; i < NNUE_L2; i++)
{
- s += g_rgfNnueFc2Weight[j][i] * rgfL2[i];
+ iSum += (INT)rgbAct2[i] * (INT)g_rgfNnueFc2Weight[j][i];
}
- rgfL3[j] = (s > 0.0f) ? s : 0.0f;
+ float s = ((float)iSum / (g_rgfNnueActScale[1] * g_rgfNnueWeightScale[1])) +
+ g_rgfNnueFc2Bias[j];
+ s = (s > 0.0f) ? s : 0.0f;
+ rgfL3[j] = s;
+ rgbAct3[j] = (UCHAR)MAX(0, MIN(255, (int)(s * g_rgfNnueActScale[2] + 0.5f)));
}
- fOut = g_rgfNnueFc3Bias;
- for (i = 0; i < NNUE_L3; i++)
+ // Stage 3: fc3, single output, no ReLU (final scalar score).
{
- fOut += g_rgfNnueFc3Weight[i] * rgfL3[i];
+ INT iSum = 0;
+ for (i = 0; i < NNUE_L3; i++)
+ {
+ iSum += (INT)rgbAct3[i] * (INT)g_rgfNnueFc3Weight[i];
+ }
+ fOut = ((float)iSum / (g_rgfNnueActScale[2] * g_rgfNnueWeightScale[2])) +
+ g_rgfNnueFc3Bias;
}
return (SCORE)(fOut * 100.0f);
}
+
+
+void
+NnueDumpActivationStats(SEARCHER_THREAD_CONTEXT *ctx)
+/**
+
+Routine description:
+
+ Diagnostic-only: re-runs NnueForward's same math but reports, per
+ layer, what fraction of quantized activations are clamped to the
+ [0,255] UCHAR rail (saturated) vs sitting strictly inside it, plus
+ the pre-clamp float stats. Built in response to the "loses a whole
+ queen and the eval barely moves" bug report, 2026-10-17 -- the
+ working theory is that ordinary middlegame positions saturate most
+ activations to the same rail regardless of material, so a single
+ piece's worth of accumulator change never survives the clamp to
+ reach the dense layers; only an extreme (near-decisive) material
+ swing moves enough units across the rail to register. This
+ function exists to confirm or refute that theory, not to fix
+ anything -- reference/measurement-only, not wired into search.
+
+Parameters:
+
+ SEARCHER_THREAD_CONTEXT *ctx
+
+Return value:
+
+ void
+
+**/
+{
+ POSITION *pos = &ctx->sPosition;
+ ULONG uStm = pos->uToMove;
+ ULONG uOther = FLIP(uStm);
+ UCHAR rgbAct1[2 * NNUE_ACCUMULATOR_SIZE];
+ UCHAR rgbAct2[NNUE_L2];
+ UCHAR rgbAct3[NNUE_L3];
+ ULONG i, j;
+ ULONG uClampLow, uClampHigh;
+ float fMin, fMax, fSum;
+
+ // Stage 0 (fc1 input) -- same math as NnueForward, but track the
+ // pre-clamp float range and how many of the 512 entries hit each
+ // rail.
+ uClampLow = uClampHigh = 0;
+ fMin = FLT_MAX;
+ fMax = -FLT_MAX;
+ fSum = 0.0f;
+ for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
+ {
+ float v = (float)ctx->sNnueAccum[uStm].rgiValue[i] / g_fNnueFtScale;
+ v = (v > 0.0f) ? v : 0.0f;
+ float q = v * g_rgfNnueActScale[0] + 0.5f;
+ if (q <= 0.0f) uClampLow++;
+ if (q >= 255.0f) uClampHigh++;
+ if (q < fMin) fMin = q;
+ if (q > fMax) fMax = q;
+ fSum += q;
+ rgbAct1[i] = (UCHAR)MAX(0, MIN(255, (int)q));
+ }
+ for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
+ {
+ float v = (float)ctx->sNnueAccum[uOther].rgiValue[i] / g_fNnueFtScale;
+ v = (v > 0.0f) ? v : 0.0f;
+ float q = v * g_rgfNnueActScale[0] + 0.5f;
+ if (q <= 0.0f) uClampLow++;
+ if (q >= 255.0f) uClampHigh++;
+ if (q < fMin) fMin = q;
+ if (q > fMax) fMax = q;
+ fSum += q;
+ rgbAct1[NNUE_ACCUMULATOR_SIZE + i] = (UCHAR)MAX(0, MIN(255, (int)q));
+ }
+ Trace("fc1 input (%u units): %u at 0, %u at 255, range [%.1f, %.1f], mean %.1f\n",
+ 2 * NNUE_ACCUMULATOR_SIZE, uClampLow, uClampHigh, fMin, fMax,
+ fSum / (float)(2 * NNUE_ACCUMULATOR_SIZE));
+
+ // Stage 1 (fc2 input)
+ uClampLow = uClampHigh = 0;
+ fMin = FLT_MAX;
+ fMax = -FLT_MAX;
+ fSum = 0.0f;
+ for (j = 0; j < NNUE_L2; j++)
+ {
+ INT iSum = 0;
+ for (i = 0; i < 2 * NNUE_ACCUMULATOR_SIZE; i++)
+ {
+ iSum += (INT)rgbAct1[i] * (INT)g_rgfNnueFc1Weight[j][i];
+ }
+ float s = ((float)iSum / (g_rgfNnueActScale[0] * g_rgfNnueWeightScale[0])) +
+ g_rgfNnueFc1Bias[j];
+ s = (s > 0.0f) ? s : 0.0f;
+ float q = s * g_rgfNnueActScale[1] + 0.5f;
+ if (q <= 0.0f) uClampLow++;
+ if (q >= 255.0f) uClampHigh++;
+ if (q < fMin) fMin = q;
+ if (q > fMax) fMax = q;
+ fSum += q;
+ rgbAct2[j] = (UCHAR)MAX(0, MIN(255, (int)q));
+ }
+ Trace("fc2 input (%u units): %u at 0, %u at 255, range [%.1f, %.1f], mean %.1f\n",
+ (ULONG)NNUE_L2, uClampLow, uClampHigh, fMin, fMax,
+ fSum / (float)NNUE_L2);
+
+ // Stage 2 (fc3 input)
+ uClampLow = uClampHigh = 0;
+ fMin = FLT_MAX;
+ fMax = -FLT_MAX;
+ fSum = 0.0f;
+ for (j = 0; j < NNUE_L3; j++)
+ {
+ INT iSum = 0;
+ for (i = 0; i < NNUE_L2; i++)
+ {
+ iSum += (INT)rgbAct2[i] * (INT)g_rgfNnueFc2Weight[j][i];
+ }
+ float s = ((float)iSum / (g_rgfNnueActScale[1] * g_rgfNnueWeightScale[1])) +
+ g_rgfNnueFc2Bias[j];
+ s = (s > 0.0f) ? s : 0.0f;
+ float q = s * g_rgfNnueActScale[2] + 0.5f;
+ if (q <= 0.0f) uClampLow++;
+ if (q >= 255.0f) uClampHigh++;
+ if (q < fMin) fMin = q;
+ if (q > fMax) fMax = q;
+ fSum += q;
+ rgbAct3[j] = (UCHAR)MAX(0, MIN(255, (int)q));
+ }
+ Trace("fc3 input (%u units): %u at 0, %u at 255, range [%.1f, %.1f], mean %.1f\n",
+ (ULONG)NNUE_L3, uClampLow, uClampHigh, fMin, fMax,
+ fSum / (float)NNUE_L3);
+}
+
+
+// Calibration-only lazy-eval probe (docs/NNUE.md's "Lazy-evaluation
+// tiering plan"). A linear regression fit offline over fc1's own
+// input (512-wide: the ReLU'd, concatenated accumulator halves) --
+// NOT a second trained net, just a cheap linear read of a
+// representation the real net already produces. Completely separate
+// weight file/format from NnueLoadWeights' production .nnue files;
+// this is measurement-only infrastructure, consulted only under
+// CALIBRATE_MARGIN_SAFETY (see eval.c's Eval()) and never used to
+// skip any real computation yet.
+float g_rgfNnueProbeMu[2 * NNUE_ACCUMULATOR_SIZE];
+float g_rgfNnueProbeSigma[2 * NNUE_ACCUMULATOR_SIZE];
+float g_rgfNnueProbeWeight[2 * NNUE_ACCUMULATOR_SIZE];
+float g_fNnueProbeBias;
+FLAG g_fNnueProbeLoaded = FALSE;
+FLAG g_fUseNnueLazyProbe = FALSE;
+
+// Runtime overrides for experimenting with how loose the NNUE lazy/
+// super-lazy margins can go without rebuilding -- 2026-10-16, see
+// docs/NNUE.md "paradox: safety doesn't win games" discussion.
+// g_iNnueLazyMarginOverride < 0 means "use eval.c's calibrated
+// per-material-bucket table as before"; >= 0 means "use this flat cp
+// value for every bucket instead". g_fUseNnueSuperLazy/
+// g_iNnueSuperLazyMargin gate a brand new, even cheaper pre-filter
+// (pure material balance, no net at all) ahead of the probe -- this
+// was previously measurement-only (see eval.c's CALIBRATE_MARGIN_SAFETY
+// block) and is now wired into the real Eval() path for the first time.
+INT g_iNnueLazyMarginOverride = -1;
+FLAG g_fUseNnueSuperLazy = FALSE;
+SCORE g_iNnueSuperLazyMargin = 0;
+
+
+FLAG
+NnueProbeLoadWeights(CHAR *szFilename)
+/**
+
+Routine description:
+
+ Loads the lazy-eval probe's weights -- a flat file written by
+ annotate_pgns/export_nnue_weights.py's sibling fitting script: a
+ 4-byte magic ("NPRB"), a uint32 dimension (must equal
+ 2*NNUE_ACCUMULATOR_SIZE), then mu/sigma/weight (each that many
+ float32s) and a single float32 bias, in that order.
+
+Parameters:
+
+ CHAR *szFilename
+
+Return value:
+
+ FLAG : TRUE on success
+
+**/
+{
+ FILE *fp;
+ CHAR szMagic[4];
+ ULONG uDim;
+
+ g_fNnueProbeLoaded = FALSE;
+ fp = fopen(szFilename, "rb");
+ if (NULL == fp)
+ {
+ Trace("NnueProbeLoadWeights: could not open %s\n", szFilename);
+ return FALSE;
+ }
+
+ if ((1 != fread(szMagic, 4, 1, fp)) ||
+ (0 != memcmp(szMagic, "NPRB", 4)))
+ {
+ Trace("NnueProbeLoadWeights: %s is not a valid probe weights "
+ "file (bad magic)\n", szFilename);
+ fclose(fp);
+ return FALSE;
+ }
+ if ((1 != fread(&uDim, sizeof(uDim), 1, fp)) ||
+ (uDim != 2 * NNUE_ACCUMULATOR_SIZE))
+ {
+ Trace("NnueProbeLoadWeights: %s dimension %lu doesn't match "
+ "this build's %lu\n", szFilename, uDim,
+ (ULONG)(2 * NNUE_ACCUMULATOR_SIZE));
+ fclose(fp);
+ return FALSE;
+ }
+
+#define READ_OR_FAIL(ptr, count) \
+ do { \
+ if ((size_t)(count) != fread((ptr), sizeof(float), (count), fp)) \
+ { \
+ Trace("NnueProbeLoadWeights: %s truncated\n", szFilename); \
+ fclose(fp); \
+ return FALSE; \
+ } \
+ } while (0)
+
+ READ_OR_FAIL(g_rgfNnueProbeMu, 2 * NNUE_ACCUMULATOR_SIZE);
+ READ_OR_FAIL(g_rgfNnueProbeSigma, 2 * NNUE_ACCUMULATOR_SIZE);
+ READ_OR_FAIL(g_rgfNnueProbeWeight, 2 * NNUE_ACCUMULATOR_SIZE);
+ READ_OR_FAIL(&g_fNnueProbeBias, 1);
+
+#undef READ_OR_FAIL
+
+ fclose(fp);
+ g_fNnueProbeLoaded = TRUE;
+ Trace("NnueProbeLoadWeights: loaded %s\n", szFilename);
+ return TRUE;
+}
+
+
+SCORE
+NnueProbeScore(SEARCHER_THREAD_CONTEXT *ctx)
+/**
+
+Routine description:
+
+ Cheap lazy-eval proxy: dequantizes+ReLUs the accumulator exactly
+ like NnueForward's stage 0 (same own-perspective-first
+ concatenation), standardizes by the probe's fitted mu/sigma, then
+ a single 512-wide dot product -- no fc1/fc2/fc3 at all. One
+ multiply-add per dimension plus the dequant/ReLU/standardize work,
+ versus NnueForward's full ~17,440 MACs across three layers --
+ see docs/NNUE.md's latency estimate.
+
+Parameters:
+
+ SEARCHER_THREAD_CONTEXT *ctx
+
+Return value:
+
+ SCORE : side-to-move-relative, in centipawns, same convention as
+ NnueForward (so the two are directly comparable)
+
+**/
+{
+ POSITION *pos = &ctx->sPosition;
+ ULONG uStm = pos->uToMove;
+ ULONG uOther = FLIP(uStm);
+ float fSum;
+ ULONG i;
+
+ fSum = g_fNnueProbeBias;
+ for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
+ {
+ float v = (float)ctx->sNnueAccum[uStm].rgiValue[i] / g_fNnueFtScale;
+ v = (v > 0.0f) ? v : 0.0f;
+ v = (v - g_rgfNnueProbeMu[i]) / g_rgfNnueProbeSigma[i];
+ fSum += v * g_rgfNnueProbeWeight[i];
+ }
+ for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
+ {
+ float v = (float)ctx->sNnueAccum[uOther].rgiValue[i] / g_fNnueFtScale;
+ v = (v > 0.0f) ? v : 0.0f;
+ v = (v - g_rgfNnueProbeMu[NNUE_ACCUMULATOR_SIZE + i]) /
+ g_rgfNnueProbeSigma[NNUE_ACCUMULATOR_SIZE + i];
+ fSum += v * g_rgfNnueProbeWeight[NNUE_ACCUMULATOR_SIZE + i];
+ }
+
+ return (SCORE)(fSum * 100.0f);
+}
diff --git a/src/search.c b/src/search.c
index 8404d9d..7d8d686 100644
--- a/src/search.c
+++ b/src/search.c
@@ -66,6 +66,7 @@ Revision History:
**/
#include "chess.h"
+#include <math.h> // DumpQSearchFutilityCalibration's sqrt
extern ULONG g_uIterateDepth;
extern FLAG g_fCanSplit[MAX_PLY_PER_SEARCH];
@@ -713,7 +714,7 @@ Search(IN SEARCHER_THREAD_CONTEXT *ctx,
uRawImproveB,
uRawEnemyImproveB);
if ((uBetaPredictedFHThousandths != 0xFFFF) &&
- (uBetaPredictedFHThousandths > 95000))
+ (uBetaPredictedFHThousandths > 96000))
{
INC(ctx->sCounters.tree.u64QuickNullMoveSuccess);
@@ -1371,6 +1372,30 @@ Return value:
static UINT64 g_uQFutTries[QFUT_GATE_COUNT][QFUT_DIST_BUCKETS][QFUT_POS_BUCKETS][EVAL_POSITIONAL_SOURCE_COUNT];
static UINT64 g_uQFutSurprises[QFUT_GATE_COUNT][QFUT_DIST_BUCKETS][QFUT_POS_BUCKETS][EVAL_POSITIONAL_SOURCE_COUNT];
+// 2026-10-16 diagnostic (see the raw-pre-clamp iFutility capture
+// above): is MAX0 clamping iFutility to 0 for most/all of a given
+// source's population, making FUTILITY_BASE_MARGIN_BY_SOURCE's exact
+// value irrelevant for that source regardless of what it's set to?
+static UINT64 g_uQFutRawPreClampCount[EVAL_POSITIONAL_SOURCE_COUNT];
+static UINT64 g_uQFutRawPreClampNonPositive[EVAL_POSITIONAL_SOURCE_COUNT];
+static INT64 g_uQFutRawPreClampSum[EVAL_POSITIONAL_SOURCE_COUNT];
+
+// 2026-10-16: mean/stdev/max of the actual surprise MAGNITUDE
+// (iScore - iAlpha, when a rejected move's diagnostic re-search
+// raised alpha after all) by source -- distinguishes "surprises are
+// consistently small (eval noise near the boundary)" from
+// "occasional huge swings (real missed tactics, e.g. a mate the
+// shallow SEE-based rejection couldn't see)". Not clamped -- a real
+// mate score legitimately belongs in this population, not an outlier
+// to exclude.
+static UINT64 g_uQFutSurpriseMagCount[EVAL_POSITIONAL_SOURCE_COUNT];
+static INT64 g_uQFutSurpriseMagSum[EVAL_POSITIONAL_SOURCE_COUNT];
+static double g_dQFutSurpriseMagSumSq[EVAL_POSITIONAL_SOURCE_COUNT];
+static INT64 g_uQFutSurpriseMagMax[EVAL_POSITIONAL_SOURCE_COUNT];
+// Cheap shape check without a full histogram: how many surprises are
+// "clearly a real tactic" (>100cp) vs "plausibly just noise" (<=100cp).
+static UINT64 g_uQFutSurpriseMagOver100[EVAL_POSITIONAL_SOURCE_COUNT];
+
static ULONG
_QFutDistanceBucket(IN SCORE iDistance)
/* iDistance: how far short of the relevant threshold this move was
@@ -1397,6 +1422,21 @@ _QFutPositionalBucket(IN SCORE iPositional)
return(3);
}
+static CHAR *g_szQFutGateNames[QFUT_GATE_COUNT] =
+{
+ "generic capture/promo",
+ "checking capture/promo (VALUE_ROOK)",
+ "quiet check (VALUE_BISHOP)",
+};
+static CHAR *g_szQFutSourceNames[EVAL_POSITIONAL_SOURCE_COUNT] =
+{
+ "full eval",
+ "regular lazy",
+ "super lazy",
+ "nnue full",
+ "nnue lazy probe",
+};
+
static void
_QFutDiagnoseReject(IN SEARCHER_THREAD_CONTEXT *ctx,
IN ULONG uMoveNum,
@@ -1466,24 +1506,54 @@ Return value:
g_uQFutTries[uGate][uDistBucket][uPosBucket][uPositionalSource]++;
if (iScore > iAlpha)
{
+ SCORE iMagnitude;
g_uQFutSurprises[uGate][uDistBucket][uPosBucket][uPositionalSource]++;
+
+ iMagnitude = (SCORE)(iScore - iAlpha);
+ g_uQFutSurpriseMagCount[uPositionalSource]++;
+ g_uQFutSurpriseMagSum[uPositionalSource] += (INT64)iMagnitude;
+ g_dQFutSurpriseMagSumSq[uPositionalSource] +=
+ (double)iMagnitude * (double)iMagnitude;
+ if ((INT64)iMagnitude > g_uQFutSurpriseMagMax[uPositionalSource])
+ {
+ g_uQFutSurpriseMagMax[uPositionalSource] = (INT64)iMagnitude;
+ }
+ if (iMagnitude > 100)
+ {
+ g_uQFutSurpriseMagOver100[uPositionalSource]++;
+ }
+
+ // 2026-10-16: a few concrete real examples, capped, so a
+ // surprise-rate gap between sources (e.g. nnue full's 7.41%
+ // vs classic full's ~1.91% in the same gate) can be looked at
+ // directly instead of only as an aggregate percentage. Only
+ // samples the two NNUE sources -- classic's own numbers are
+ // already understood from the original derivation.
+ if ((uPositionalSource == EVAL_POSITIONAL_SOURCE_NNUE_FULL) ||
+ (uPositionalSource == EVAL_POSITIONAL_SOURCE_NNUE_LAZY))
+ {
+ static ULONG uSampleCount[EVAL_POSITIONAL_SOURCE_COUNT];
+ if (uSampleCount[uPositionalSource] < 10)
+ {
+ CHAR *pszFen = PositionToFen(&ctx->sPosition);
+ if (NULL != pszFen)
+ {
+ uSampleCount[uPositionalSource]++;
+ Trace("QFUT_SURPRISE_SAMPLE source=%s gate=%s "
+ "move=0x%lx iAlpha=%d iScore=%d iDistance=%d "
+ "fen=%s\n",
+ g_szQFutSourceNames[uPositionalSource],
+ g_szQFutGateNames[uGate],
+ mv.uMove, (int)iAlpha, (int)iScore, (int)iDistance,
+ pszFen);
+ SystemFreeMemory(pszFen);
+ }
+ }
+ }
}
}
-static CHAR *g_szQFutGateNames[QFUT_GATE_COUNT] =
-{
- "generic capture/promo",
- "checking capture/promo (VALUE_ROOK)",
- "quiet check (VALUE_BISHOP)",
-};
-static CHAR *g_szQFutSourceNames[EVAL_POSITIONAL_SOURCE_COUNT] =
-{
- "full eval",
- "regular lazy",
- "super lazy",
-};
-
void
DumpQSearchFutilityCalibration(void)
/**
@@ -1612,6 +1682,51 @@ Return value:
}
}
}
+
+ Trace("\nSurprise magnitude (iScore - iAlpha, when a rejected move's "
+ "diagnostic re-search raised alpha) by source:\n");
+ for (s = 0; s < EVAL_POSITIONAL_SOURCE_COUNT; s++)
+ {
+ UINT64 u64Count = g_uQFutSurpriseMagCount[s];
+ double dMean, dVariance, dStdev;
+ if (0 == u64Count)
+ {
+ Trace(" %s: no surprises\n", g_szQFutSourceNames[s]);
+ continue;
+ }
+ dMean = (double)g_uQFutSurpriseMagSum[s] / (double)u64Count;
+ dVariance = (g_dQFutSurpriseMagSumSq[s] / (double)u64Count) -
+ (dMean * dMean);
+ if (dVariance < 0.0)
+ {
+ dVariance = 0.0;
+ }
+ dStdev = sqrt(dVariance);
+ Trace(" %-16s: n=%8" COMPILER_LONGLONG_UNSIGNED_FORMAT
+ " mean=%7.1f stdev=%7.1f max=%6" COMPILER_LONGLONG_UNSIGNED_FORMAT
+ " >100cp=%5.2f%%\n",
+ g_szQFutSourceNames[s], u64Count, dMean, dStdev,
+ (UINT64)g_uQFutSurpriseMagMax[s],
+ 100.0 * (double)g_uQFutSurpriseMagOver100[s] / (double)u64Count);
+ }
+
+ Trace("\nRaw pre-clamp iFutility by source (is MAX0 saturating this "
+ "source's population, making the margin constant moot?):\n");
+ for (s = 0; s < EVAL_POSITIONAL_SOURCE_COUNT; s++)
+ {
+ UINT64 u64Count = g_uQFutRawPreClampCount[s];
+ if (0 == u64Count)
+ {
+ Trace(" %s: no samples\n", g_szQFutSourceNames[s]);
+ continue;
+ }
+ Trace(" %-16s: n=%10" COMPILER_LONGLONG_UNSIGNED_FORMAT
+ " mean=%7.1f clamped(<=0)=%5.2f%%\n",
+ g_szQFutSourceNames[s], u64Count,
+ (double)g_uQFutRawPreClampSum[s] / (double)u64Count,
+ 100.0 * (double)g_uQFutRawPreClampNonPositive[s] /
+ (double)u64Count);
+ }
}
#endif // CALIBRATE_QSEARCH_FUTILITY
@@ -1891,6 +2006,8 @@ static const SCORE FUTILITY_BASE_MARGIN_BY_SOURCE[EVAL_POSITIONAL_SOURCE_COUNT]
FUTILITY_BASE_MARGIN_FULL, // EVAL_POSITIONAL_SOURCE_FULL
FUTILITY_BASE_MARGIN_LAZY, // EVAL_POSITIONAL_SOURCE_LAZY
FUTILITY_BASE_MARGIN_SUPERLAZY, // EVAL_POSITIONAL_SOURCE_SUPERLAZY
+ FUTILITY_BASE_MARGIN_NNUE_FULL, // EVAL_POSITIONAL_SOURCE_NNUE_FULL
+ FUTILITY_BASE_MARGIN_NNUE_LAZY, // EVAL_POSITIONAL_SOURCE_NNUE_LAZY
};
@@ -2292,6 +2409,23 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx,
iFutility = iAlpha -
(FUTILITY_BASE_MARGIN_BY_SOURCE[pi->uLastPositionalSource] +
rgiPositional[pos->uToMove]) - iEval;
+#ifdef CALIBRATE_QSEARCH_FUTILITY
+ // 2026-10-16 diagnostic: is iFutility actually being clamped
+ // to 0 by MAX0 below for this source, making the margin
+ // constant irrelevant regardless of its value? See the
+ // investigation into NNUE_FULL's margin change having zero
+ // measured effect.
+ {
+ ULONG uSrc = pi->uLastPositionalSource;
+ ASSERT(uSrc < EVAL_POSITIONAL_SOURCE_COUNT);
+ g_uQFutRawPreClampCount[uSrc]++;
+ g_uQFutRawPreClampSum[uSrc] += (INT64)iFutility;
+ if (iFutility <= 0)
+ {
+ g_uQFutRawPreClampNonPositive[uSrc]++;
+ }
+ }
+#endif
iFutility = MAX0(iFutility);
#endif
}
diff --git a/src/split_ep.py b/src/split_ep.py
new file mode 100755
index 0000000..557dbec
--- /dev/null
+++ b/src/split_ep.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Split a script-format .ep_ file (setboard/solution/id/go blocks, as
+used by typhoon's `script` command) into two disjoint, shuffled halves
+for a train/holdout threshold sweep.
+
+Usage: split_ep.py <input.ep_> <out_a.ep_> <out_b.ep_> [--seed N]
+
+Splits by whole position blocks (never mid-position), shuffled with a
+fixed seed so the split is reproducible but not biased by the file's
+original ordering (ECM.ep_ is not randomly ordered).
+"""
+import argparse
+import random
+import sys
+
+
+def read_blocks(path):
+ """Each block is the lines from one 'setboard' up to and including
+ the following 'go' line."""
+ blocks = []
+ current = []
+ with open(path) as f:
+ for line in f:
+ stripped = line.rstrip("\n")
+ if stripped.startswith("setboard") and current:
+ blocks.append(current)
+ current = []
+ current.append(stripped)
+ if current:
+ blocks.append(current)
+ return blocks
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("input")
+ ap.add_argument("out_a")
+ ap.add_argument("out_b")
+ ap.add_argument("--seed", type=int, default=0)
+ args = ap.parse_args()
+
+ blocks = read_blocks(args.input)
+ if not blocks:
+ print(f"ERROR: no position blocks found in {args.input}", file=sys.stderr)
+ sys.exit(1)
+
+ rng = random.Random(args.seed)
+ order = list(range(len(blocks)))
+ rng.shuffle(order)
+
+ mid = len(order) // 2
+ half_a = sorted(order[:mid])
+ half_b = sorted(order[mid:])
+
+ with open(args.out_a, "w") as f:
+ for i in half_a:
+ f.write("\n".join(blocks[i]) + "\n")
+
+ with open(args.out_b, "w") as f:
+ for i in half_b:
+ f.write("\n".join(blocks[i]) + "\n")
+
+ print(f"{len(blocks)} positions in {args.input} -> "
+ f"{len(half_a)} in {args.out_a}, {len(half_b)} in {args.out_b} "
+ f"(seed={args.seed})")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/testnnue.c b/src/testnnue.c
index fdfaf35..4ee516d 100644
--- a/src/testnnue.c
+++ b/src/testnnue.c
@@ -30,6 +30,12 @@ Abstract:
afterward so this test has no side effect on anything that runs
after it in the same process.
+ g_rgfNnueWeight/g_rgfNnueBias are SHORT (quantized) and
+ NNUE_ACCUMULATOR.rgiValue is INT -- all integer arithmetic, so
+ incremental-vs-recompute comparison is exact (no float-rounding
+ tolerance needed, unlike the original float-accumulator version
+ of this test).
+
Author:
Scott Gasch ([email protected]) 15 Oct 2026
@@ -59,12 +65,12 @@ Routine description:
for (j = 0; j < NNUE_ACCUMULATOR_SIZE; j++)
{
g_rgfNnueWeight[i][j] =
- ((float)(i % 97) * 0.1f) + ((float)(j % 13) * 0.01f) + 1.0f;
+ (SHORT)(((i % 97) * 10) + ((j % 13) * 1) + 100);
}
}
for (j = 0; j < NNUE_ACCUMULATOR_SIZE; j++)
{
- g_rgfNnueBias[j] = (float)(j % 7) * 0.5f;
+ g_rgfNnueBias[j] = (SHORT)((j % 7) * 50);
}
}
@@ -95,12 +101,10 @@ Routine description:
against ctx->sPosition and compares it element-by-element against
the incrementally-maintained ctx->sNnueAccum. Panics (via
UtilPanic, same convention as every other self-test) on any
- mismatch outside a small float-rounding tolerance -- real
- indexing/logic bugs produce large deltas (a whole missing or
- extra ~1-100-magnitude weight row), not rounding noise, so the
- tolerance only needs to be loose enough to absorb summation-order
- differences between the two paths, not loose enough to hide a
- real bug.
+ mismatch. Integer arithmetic throughout (SHORT table, INT
+ accumulator) means incremental and from-scratch recompute must
+ match EXACTLY -- no rounding tolerance needed or wanted; any
+ nonzero delta is a real indexing/logic bug.
Parameters:
@@ -114,7 +118,6 @@ Return value:
**/
{
- static const float EPSILON = 0.01f;
NNUE_ACCUMULATOR sFresh;
ULONG uColor;
ULONG i;
@@ -124,16 +127,16 @@ Return value:
NnueAccumRecompute(&sFresh, &ctx->sPosition, uColor);
for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++)
{
- float fDelta = sFresh.rgfValue[i] - ctx->sNnueAccum[uColor].rgfValue[i];
- if ((fDelta > EPSILON) || (fDelta < -EPSILON))
+ INT iDelta = sFresh.rgiValue[i] - ctx->sNnueAccum[uColor].rgiValue[i];
+ if (iDelta != 0)
{
Trace("TestNnueAccumulator: MISMATCH at %s, color=%lu, "
- "index=%lu: incremental=%f recompute=%f "
+ "index=%lu: incremental=%ld recompute=%ld "
"mv.uMove=0x%lx cFrom=0x%x cTo=0x%x pMoved=%u "
"pCaptured=%u pPromoted=%u bvFlags=0x%x FEN=%s\n",
szLabel, uColor, i,
- ctx->sNnueAccum[uColor].rgfValue[i],
- sFresh.rgfValue[i],
+ (long)ctx->sNnueAccum[uColor].rgiValue[i],
+ (long)sFresh.rgiValue[i],
mv.uMove, mv.cFrom, mv.cTo, mv.pMoved,
mv.pCaptured, mv.pPromoted, mv.bvFlags,
PositionToFen(&ctx->sPosition));
diff --git a/src/util.c b/src/util.c
index 23a1647..0192f20 100644
--- a/src/util.c
+++ b/src/util.c
@@ -1019,7 +1019,6 @@ FinishPVTailFromHash(SEARCHER_THREAD_CONTEXT *ctx,
{
if (u64VisitedSigs[u] == u64Sig)
{
- if (uLenRemain > 6) strcat(buf, "<REP>");
goto unmake;
}
}
@@ -1369,7 +1368,9 @@ Return value:
// running under xboard, match its expected format. Otherwise
// be more helpful / verbose.
//
- if ((dNow > 0.35) || ((dNow > 0.15) && (g_Options.fVerbosePosting)))
+ if ((dNow >= g_Options.dPostThrottleSec) ||
+ ((dNow > (0.15 / 0.35) * g_Options.dPostThrottleSec) &&
+ (TRUE == g_Options.fVerbosePosting)))
{
if (TRUE == g_Options.fRunningUnderXboard)
{
diff --git a/src/vars.c b/src/vars.c
index 9d736ab..ee2b29c 100644
--- a/src/vars.c
+++ b/src/vars.c
@@ -143,6 +143,10 @@ USER_VARIABLE g_UserVarList[] =
"B",
(void *)&(g_Options.fShouldPost),
NULL },
+ { "PostThrottle",
+ "D",
+ (void *)&(g_Options.dPostThrottleSec),
+ NULL },
{ "ResignThreshold",
"I",
(void *)&(g_Options.iResignThreshold),
@@ -442,7 +446,11 @@ Return value:
y = atoi(argv[2]);
*((int *)g_UserVarList[x].pValue) = y;
break;
-
+
+ case 'D':
+ *((double *)g_UserVarList[x].pValue) = atof(argv[2]);
+ break;
+
case 'B':
if (toupper(*(argv[2])) == 'T')
{