diff options
| author | Scott Gasch <[email protected]> | 2026-10-20 09:26:44 -0700 |
|---|---|---|
| committer | Scott Gasch <[email protected]> | 2026-10-20 09:26:44 -0700 |
| commit | c8c6bbc2c1a103e5b07b079a0f8bdf66c7724d80 (patch) | |
| tree | b6c2137c9ac577fa83b1e948fa0311d53dfe445d /src | |
| parent | 3b5d3677bb14ae14728a1c4b221176bb83667d92 (diff) | |
Wire NNUE eval into search, build the training pipeline, and add a fast
move/eval-agreement benchmark against Stockfish (tests/lop)
Implements the architecture decisions in src/docs/NNUE.md: HalfKAv2_hm
feature set, int16/int8 quantization, per-ply accumulator backup for
king moves, and a lazy-eval-ready integration into search.c/eval.c.
Adds the full annotate_pgns/ training pipeline (extraction, K-fit/
labeling, quiet filtering, tensor building, training, export) plus an
end-to-end orchestrator (run_nnue_pipeline.py) and validation tooling
(sanity_check_net.py, compare_vs_stockfish_net.py). Adds tests/lop:
a 10k-position list-of-positions benchmark comparing engine eval rank
and move choice against pre-computed Stockfish/lc0 references, much
cheaper than a full self-play match for day-to-day progress checks.
First sd10-matched self-play result against classic HCE: LLR +2.28
(H1=+2.94), -72=119+169 over 360 games, confirming the net is
"smarter per node" at matched depth. A real-time (--st 4) match is
currently weaker (candidate loses ~3 plies of depth to a ~3.3-3.7x
nps gap against an unrecalibrated search), consistent with
NNUE.md's adoption-criterion section -- pruning/reduction constants
still need retuning for the new eval's score scale before a fair
real-time comparison is possible.
Co-Authored-By: Claude Sonnet 5 <[email protected]>
Diffstat (limited to 'src')
| -rw-r--r-- | src/GNUmakefile | 14 | ||||
| -rw-r--r-- | src/TODO | 6 | ||||
| -rwxr-xr-x | src/chess.h | 163 | ||||
| -rw-r--r-- | src/command.c | 196 | ||||
| -rw-r--r-- | src/docs/NNUE.md | 1605 | ||||
| -rw-r--r-- | src/eval.c | 347 | ||||
| -rw-r--r-- | src/eval_tune/build_cross_sectional_all.py | 228 | ||||
| -rw-r--r-- | src/eval_tune/compare_vs_crafty.py | 597 | ||||
| -rwxr-xr-x | src/eval_tune/match_play.py | 275 | ||||
| -rwxr-xr-x | src/eval_tune/nnue_vs_head.sh | 219 | ||||
| -rw-r--r-- | src/eval_tune/relabel_selfplay_pgn.py | 112 | ||||
| -rwxr-xr-x | src/eval_tune/stockfish_training_corpus.sh | 13 | ||||
| -rwxr-xr-x | src/eval_tune/test_vs_head.sh | 94 | ||||
| -rwxr-xr-x | src/eval_tune/typhoon_hce_training_corpus.sh | 173 | ||||
| -rwxr-xr-x | src/eval_tune/typhoon_nnue_training_corpus.sh | 224 | ||||
| -rw-r--r-- | src/main.c | 48 | ||||
| -rw-r--r-- | src/nnueaccum.c | 30 | ||||
| -rw-r--r-- | src/nnueeval.c | 406 | ||||
| -rw-r--r-- | src/search.c | 162 | ||||
| -rwxr-xr-x | src/split_ep.py | 69 | ||||
| -rw-r--r-- | src/testnnue.c | 31 | ||||
| -rw-r--r-- | src/util.c | 5 | ||||
| -rw-r--r-- | src/vars.c | 10 |
23 files changed, 4818 insertions, 209 deletions
diff --git a/src/GNUmakefile b/src/GNUmakefile index bbe79d1..7dbb143 100644 --- a/src/GNUmakefile +++ b/src/GNUmakefile @@ -98,6 +98,20 @@ ifdef GPROF PROFILE += -pg endif +# Scoped ISA opt-in for NNUE SIMD work (int8/int16 dot products, and +# the FC head's existing float32 loops which clang already +# autovectorizes to SSE only without this). Default ON since 2026-10-16 +# -- confirmed a real, measured win for both classic HCE (~3%) and +# (once real NNUE weights are loaded) the NNUE FC head (~7%, see +# docs/NNUE.md), and the build box (Zen 2) has full-width 256-bit +# AVX2 + FMA3. Pass AVX2=0 explicitly to disable (e.g. building on a +# pre-Haswell/pre-Zen box, 2013 and earlier) -- unlike -march=native, +# this doesn't assume the BUILD box matches the RUN box, but it does +# assume AVX2 availability by default now rather than opt-in. +ifneq ($(AVX2),0) + PROFILE += -mavx2 -mfma +endif + ifdef DEBUG PROFILE += -g -DDEBUG BINARY = _typhoon @@ -71,3 +71,9 @@ Valgrind in debug_smoke_test.sh. Remove all line-ending whitespace in one commit. +Fix the input queue --batch mode one thing after the script; like this breaks it. + +Add nullmoves skipped to the output. Redo the output in general. + +Remove "pure Newton's square root method" stuff and get math.h loaded. + diff --git a/src/chess.h b/src/chess.h index cfe1d4e..3e65e7f 100755 --- a/src/chess.h +++ b/src/chess.h @@ -1630,22 +1630,25 @@ MOVE_SCORING_SCHEME; // ever applied where the accumulator is actually read (the FC head), // never where it's maintained. // -// Deliberately float, not yet the int16 the final architecture -// decision calls for (NNUE.md's "Quantization" section) -- that needs -// a calibrated per-layer scale from the quantization-aware export -// pass, which doesn't exist yet. This branch's job is proving the -// incremental-maintenance plumbing (king-move backup/restore nesting, -// capture/promotion edge cases) is correct; swapping the element type -// is a later, decoupled, mechanical follow-up once real quantized -// weights exist -- not something to conflate with debugging the -// harder plumbing now. +// Int32, not float: the embedding table (g_rgfNnueWeight) is +// quantized to SHORT at FT_SCALE (see export_nnue_weights.py / +// g_fNnueFtScale), and summing up to 32 active features' SHORT rows +// can exceed SHORT's range even though each individual row value +// fits -- measured worst-case real accumulator magnitude (NNUE.md's +// "Quantization safety-checked" note) is ~26 in float-equivalent +// units, i.e. ~106,500 at the chosen scale, well past MAX_SHORT. +// INT headroom makes this a non-issue. Converted back to float +// (divided by g_fNnueFtScale) only where the accumulator is actually +// read (the FC head, nnueeval.c's NnueForward) -- never where it's +// maintained, so incremental MakeMove/UnmakeMove updates stay purely +// additive integer arithmetic (exact, no float drift). #define NNUE_ACCUMULATOR_SIZE (256) // L1, nnue_model.py #define NNUE_L2 (32) // nnue_model.py's L2 #define NNUE_L3 (32) // nnue_model.py's L3 typedef struct _NNUE_ACCUMULATOR { - float rgfValue[NNUE_ACCUMULATOR_SIZE]; + INT rgiValue[NNUE_ACCUMULATOR_SIZE]; } NNUE_ACCUMULATOR; @@ -2127,6 +2130,18 @@ typedef struct _GAME_OPTIONS FLAG fStatusLine; FLAG fFastScript; INT iResignThreshold; + // Minimum elapsed time (seconds, since this move's search started) + // before UtilPrintPV (util.c) will emit a PV line at all -- default + // 0.35 matches the historical hardcoded constant. 2026-10-16: a + // search that finishes a move's ENTIRE iterative-deepening loop + // faster than this (common at low sd/st, e.g. scripted self-play + // corpus generation) gets NO PV line at all, not a delayed one -- + // match_play.py's "no eval captured for this move" case. Exposed as + // a runtime "set PostThrottle <seconds>" variable (vars.c) rather + // than hand-editing the constant, specifically so a fast scripted + // run can set it to 0 without changing default interactive/xboard + // behavior or risking a forgotten permanent edit. + double dPostThrottleSec; enum { @@ -2307,13 +2322,20 @@ NnueFeatureIndex(ULONG uPerspective, COOR cKingOwn, COOR cPieceSquare, PIECE pPi return idx; } -// Placeholder embedding table -- NOT yet loaded from a real trained -// checkpoint (see docs/NNUE.md's "not yet built: a weight-loading -// mechanism"). Zero-initialized by C's own static/global default, so -// every accumulator value is 0 until real weights are loaded; this is -// enough to prove the incremental-maintenance plumbing is correct. -extern float g_rgfNnueWeight[NNUE_DIMENSIONS][NNUE_ACCUMULATOR_SIZE]; -extern float g_rgfNnueBias[NNUE_ACCUMULATOR_SIZE]; +// Embedding table, quantized to SHORT (see docs/NNUE.md's +// "Quantization plan" -- this is the per-move hot path table that +// dominates memory bandwidth; halving it from float to SHORT is the +// whole point). Zero-initialized by C's own static/global default +// until a real exported checkpoint is loaded via NnueLoadWeights. +// g_fNnueFtScale is the fixed-point scale applied by +// export_nnue_weights.py's QuantizeFt before rounding to SHORT; +// NnueForward divides it back out when converting the accumulator +// (NNUE_ACCUMULATOR.rgiValue, INT) to float for the FC head. Defaults +// to 1.0f so the all-zero placeholder table behaves sanely (0/1.0 == +// 0) before any real weights are loaded. +extern SHORT g_rgfNnueWeight[NNUE_DIMENSIONS][NNUE_ACCUMULATOR_SIZE]; +extern SHORT g_rgfNnueBias[NNUE_ACCUMULATOR_SIZE]; +extern float g_fNnueFtScale; void NnueAccumAddFeature(NNUE_ACCUMULATOR *pAccum, ULONG uPerspective, @@ -2336,12 +2358,26 @@ NnueAccumRecompute(NNUE_ACCUMULATOR *pAccum, POSITION *pos, ULONG uPerspective); // (callers should, before trusting its output for anything beyond // plumbing validation). // -extern float g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE]; +// FC head weights, quantized to CHAR (int8, signed) per +// docs/NNUE.md's "Quantization plan" step 2 -- biases stay float32 +// (tiny arrays, added post-dequant, not worth quantizing). +// g_rgfNnueActScale[0..2] (AS1,AS2,AS3) are the per-stage uint8 +// fixed-point scales used to requantize each ReLU'd activation before +// the next layer's int8 dot product; g_rgfNnueWeightScale[0..2] +// (WS1,WS2,WS3) are the matching per-layer weight scales. Both are +// read from the export file's header (NnueLoadWeights) -- see +// annotate_pgns/export_nnue_weights.py's CalibrateActivationScales, +// which derives AS1-3 from real sampled positions (not a theoretical +// bound) since these activations' true range isn't knowable from the +// weights alone. +extern CHAR g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE]; extern float g_rgfNnueFc1Bias[NNUE_L2]; -extern float g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2]; +extern CHAR g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2]; extern float g_rgfNnueFc2Bias[NNUE_L3]; -extern float g_rgfNnueFc3Weight[NNUE_L3]; +extern CHAR g_rgfNnueFc3Weight[NNUE_L3]; extern float g_rgfNnueFc3Bias; +extern float g_rgfNnueActScale[3]; +extern float g_rgfNnueWeightScale[3]; extern FLAG g_fNnueWeightsLoaded; extern FLAG g_fUseNnueEval; @@ -2351,6 +2387,41 @@ NnueLoadWeights(CHAR *szFilename); SCORE NnueForward(SEARCHER_THREAD_CONTEXT *ctx); +void +NnueDumpActivationStats(SEARCHER_THREAD_CONTEXT *ctx); + +// Calibration-only lazy-eval probe (docs/NNUE.md's "Lazy-evaluation +// tiering plan") -- a linear regression fit over fc1's own input (the +// ReLU'd, concatenated accumulator), separate from the production +// .nnue weight file/format entirely. NnueProbeScore is cheap (one +// 512-wide dot product, no fc1/fc2/fc3) and exists purely to measure +// how well it tracks NnueForward's real output, via the +// CALIBRATE_MARGIN_SAFETY harness in eval.c -- not yet used to skip +// any real computation. +extern FLAG g_fNnueProbeLoaded; + +// Separate from g_fUseNnueEval/g_fNnueProbeLoaded -- controls whether +// Eval()'s NNUE short-circuit actually TRUSTS the probe-based gate to +// skip NnueForward (vs. just measuring it under CALIBRATE_MARGIN_ +// SAFETY). Defaults FALSE, same convention as g_fUseNnueEval itself -- +// an explicit opt-in, not auto-enabled just because weights/probe are +// loaded. See docs/NNUE.md's "Lazy-evaluation tiering plan". +extern FLAG g_fUseNnueLazyProbe; + +// Runtime experimentation knobs -- see nnueeval.c's comment. Flat +// override for the lazy probe's margin (< 0 = use the calibrated +// per-bucket table) and the super-lazy (material-only, no net) +// pre-filter's own on/off + margin. +extern INT g_iNnueLazyMarginOverride; +extern FLAG g_fUseNnueSuperLazy; +extern SCORE g_iNnueSuperLazyMargin; + +FLAG +NnueProbeLoadWeights(CHAR *szFilename); + +SCORE +NnueProbeScore(SEARCHER_THREAD_CONTEXT *ctx); + #ifdef PERF_COUNTERS #define INC(x) ((x) += 1) #else @@ -3925,7 +3996,19 @@ IsDraw(SEARCHER_THREAD_CONTEXT *ctx); #define EVAL_POSITIONAL_SOURCE_FULL (0) #define EVAL_POSITIONAL_SOURCE_LAZY (1) #define EVAL_POSITIONAL_SOURCE_SUPERLAZY (2) -#define EVAL_POSITIONAL_SOURCE_COUNT (3) +// Added 2026-10-16 (docs/NNUE.md's "Lazy-evaluation tiering plan"): +// Eval()'s NNUE short-circuit never set uLastPositionalSource at all +// before this -- every NNUE-sourced score silently inherited whatever +// stale value a prior classic-Eval() call left in that ply's PLY_INFO +// slot (or the C zero-init default, EVAL_POSITIONAL_SOURCE_FULL, +// if none ever ran), so QSearch futility pruning was using an +// unexamined, accidentally-conservative margin for every NNUE score, +// with no distinction between the full forward pass (exact) and the +// lazy accumulator-probe exit (real, measured error). These two give +// each NNUE source its own tier, same as the three classic ones. +#define EVAL_POSITIONAL_SOURCE_NNUE_FULL (3) +#define EVAL_POSITIONAL_SOURCE_NNUE_LAZY (4) +#define EVAL_POSITIONAL_SOURCE_COUNT (5) // Per-tier qsearch futility base margins -- see search.c's // FUTILITY_BASE_MARGIN_BY_SOURCE for the full derivation. Named here @@ -3967,6 +4050,44 @@ IsDraw(SEARCHER_THREAD_CONTEXT *ctx); // qsearchfutility` if iPositional's construction changes again. #define FUTILITY_BASE_MARGIN_LAZY (350) #define FUTILITY_BASE_MARGIN_SUPERLAZY (175) +// Placeholder starting points, 2026-10-16 -- NOT yet calibrated via +// `calibrate qsearchfutility` against real data (see +// EVAL_POSITIONAL_SOURCE_NNUE_FULL/NNUE_LAZY's own comment). NNUE_FULL +// starts equal to classic FULL (both are exact, non-approximate static +// evals -- a reasonable prior pending real measurement); NNUE_LAZY +// starts equal to classic LAZY (both are an approximate exit one tier +// down from exact). Re-derive both for real before trusting them. +// 2026-10-16: first real data (twic_sample.ep_, 1500 positions, sd 6) +// showed NNUE_FULL's 7.41% surprise rate at 450 in the checking- +// capture/promo gate. Widening to 650 was then tried and tested as +// NO-OP (identical 7.41%/n) -- root-caused to iFutility's MAX0 clamp +// saturating this source's population 100.00% of the time (mean +// pre-clamp -589.3) -- the margin constant is never actually read for +// ~all of this test pool, so neither 450 nor 650 is "validated," both +// are simply untested by this data. Reverted to 450 (matches classic +// FULL) as the more defensible default pending real evidence either +// way: when NNUE_FULL surprises DO occur (in whatever small +// non-saturated slice exists, not yet isolated), their magnitude runs +// smaller than classic FULL's own surprises (mean 477.5 vs 938.7cp, +// >100cp 9.88% vs 28.74%, twic_sample.ep_ sd6) -- no evidence NNUE's +// rare misses are more expensive, so no reason to prefer a wider, +// unvalidated number over HCE's own long-trusted one. +// +// TODO: find and calibrate against the actual non-saturated NNUE_FULL +// population specifically (likely nodes where iEval sits close to +// iAlpha rather than comfortably above it -- the saturated case is +// "iEval way above iAlpha", so look at the complement) before trusting +// any NNUE_FULL margin as real, NNUE-specific-validated rather than a +// borrowed HCE default. +// +// NNUE_LAZY=200 (tightened from 350) IS genuinely validated -- real +// headroom confirmed (only 24.76% of its population even hits the +// clamp, unlike NNUE_FULL's 100%), surprise rate landed in a sane +// 1.3-2.1% range on real data, and its own surprise magnitudes +// (mean 5085.3, >100cp 19.45%) run well below classic regular lazy's +// (mean 11030.2, >100cp 69.73%). Keep this one. +#define FUTILITY_BASE_MARGIN_NNUE_FULL (450) +#define FUTILITY_BASE_MARGIN_NNUE_LAZY (200) // Measured: disabling this entirely (see lmr_testing/RESULTS.md) is a // clear net loss across ringers/confident_quick/hard_quick, so IID itself diff --git a/src/command.c b/src/command.c index 6e11d90..1767e98 100644 --- a/src/command.c +++ b/src/command.c @@ -819,9 +819,13 @@ Return value: if (argc < 2) { Trace("Usage: nnue load <filename>\n" + " nnue probeload <filename>\n" " nnue eval\n" + " nnue stats\n" " nnue on\n" - " nnue off\n"); + " nnue off\n" + " nnue lazyprobeon\n" + " nnue lazyprobeoff\n"); return; } @@ -843,6 +847,28 @@ Return value: return; } + if (!STRCMPI(argv[1], "probeload")) + { + // Loads the experimental lazy-eval probe's weights (docs/ + // NNUE.md's "Lazy-evaluation tiering plan") -- a separate, + // calibration-only weight file, not the production .nnue + // format NnueLoadWeights reads. + if (argc < 3) + { + Trace("Error (missing argument)\n"); + return; + } + if (!NnueProbeLoadWeights(argv[2])) + { + Trace("Error loading NNUE probe weights file.\n"); + } + else + { + Trace("Loaded NNUE probe weights from \"%s\"\n", argv[2]); + } + return; + } + if (!STRCMPI(argv[1], "eval")) { if (!g_fNnueWeightsLoaded) @@ -880,6 +906,34 @@ Return value: return; } + if (!STRCMPI(argv[1], "stats")) + { + // Diagnostic: per-layer activation saturation report. See + // NnueDumpActivationStats's routine description (nnueeval.c). + if (!g_fNnueWeightsLoaded) + { + Trace("Error: no NNUE weights loaded -- " + "try 'nnue load <file>' first.\n"); + return; + } + if (NULL == ctx) + { + ctx = SystemAllocateMemory(sizeof(SEARCHER_THREAD_CONTEXT)); + if (NULL == ctx) + { + Trace("Out of memory.\n"); + return; + } + InitializeSearcherContext(pos, ctx); + } + else + { + ReInitializeSearcherContext(pos, ctx); + } + NnueDumpActivationStats(ctx); + return; + } + if (!STRCMPI(argv[1], "on")) { if (!g_fNnueWeightsLoaded) @@ -900,10 +954,148 @@ Return value: return; } + if (!STRCMPI(argv[1], "lazyprobeon")) + { + if (!g_fNnueProbeLoaded) + { + Trace("Error: no NNUE probe weights loaded -- " + "try 'nnue probeload <file>' first.\n"); + return; + } + g_fUseNnueLazyProbe = TRUE; + Trace("NNUE lazy probe gate enabled.\n"); + return; + } + + if (!STRCMPI(argv[1], "lazyprobeoff")) + { + g_fUseNnueLazyProbe = FALSE; + Trace("NNUE lazy probe gate disabled.\n"); + return; + } + + if (!STRCMPI(argv[1], "lazymargin")) + { + // Flat override for every material bucket's lazy-probe margin + // -- experimentation knob, see nnueeval.c. Pass a negative + // number (or omit) to go back to the calibrated table. + if (argc < 3) + { + g_iNnueLazyMarginOverride = -1; + Trace("NNUE lazy margin override cleared (back to calibrated table).\n"); + return; + } + g_iNnueLazyMarginOverride = atoi(argv[2]); + Trace("NNUE lazy margin override set to %d cp.\n", g_iNnueLazyMarginOverride); + return; + } + + if (!STRCMPI(argv[1], "superlazyon")) + { + if (argc < 3) + { + Trace("Usage: nnue superlazyon <margin cp>\n"); + return; + } + g_iNnueSuperLazyMargin = (SCORE)atoi(argv[2]); + g_fUseNnueSuperLazy = TRUE; + Trace("NNUE super-lazy (material-only) pre-filter enabled, margin=%d cp.\n", + g_iNnueSuperLazyMargin); + return; + } + + if (!STRCMPI(argv[1], "superlazyoff")) + { + g_fUseNnueSuperLazy = FALSE; + Trace("NNUE super-lazy pre-filter disabled.\n"); + return; + } + + if (!STRCMPI(argv[1], "bench")) + { + // Scratch diagnostic, not a shipped feature -- times + // NnueAccumAddFeature/RemoveFeature (the per-MakeMove hot + // path) against NnueForward/NnueProbeScore (the per-eval hot + // path) so we know which one actually dominates before + // spending effort hand-vectorizing either. See docs/NNUE.md. + const ULONG N = 2000000; + ULONG i; + UINT64 u64Start, u64AddRemove, u64Forward, u64Probe; + NNUE_ACCUMULATOR acc; + COOR cKing = pos->cKingSq[WHITE]; + COOR cSq = FILE_RANK_TO_COOR(3, 4); + PIECE p = pos->rgSquare[FILE_RANK_TO_COOR(0, 1)].pPiece; + + memset(&acc, 0, sizeof(acc)); + u64Start = SystemReadTimeStampCounter(); + for (i = 0; i < N; i++) + { + NnueAccumAddFeature(&acc, WHITE, cKing, cSq, p); + NnueAccumRemoveFeature(&acc, WHITE, cKing, cSq, p); + } + u64AddRemove = SystemReadTimeStampCounter() - u64Start; + + if (!g_fNnueWeightsLoaded) + { + Trace("nnue bench: load weights first ('nnue load <file>') " + "to also time the forward pass.\n"); + Trace("Add+Remove (x%lu): %" COMPILER_LONGLONG_UNSIGNED_FORMAT + " cycles total, %.1f cycles/call\n", + N, u64AddRemove, (double)u64AddRemove / (2.0 * N)); + return; + } + + if (NULL == ctx) + { + ctx = SystemAllocateMemory(sizeof(SEARCHER_THREAD_CONTEXT)); + InitializeSearcherContext(pos, ctx); + } + else + { + ReInitializeSearcherContext(pos, ctx); + } + + u64Start = SystemReadTimeStampCounter(); + for (i = 0; i < N; i++) + { + volatile SCORE s = NnueForward(ctx); + (void)s; + } + u64Forward = SystemReadTimeStampCounter() - u64Start; + + u64Probe = 0; + if (g_fNnueProbeLoaded) + { + u64Start = SystemReadTimeStampCounter(); + for (i = 0; i < N; i++) + { + volatile SCORE s = NnueProbeScore(ctx); + (void)s; + } + u64Probe = SystemReadTimeStampCounter() - u64Start; + } + + Trace("nnue bench (N=%lu):\n", N); + Trace(" Add+Remove pair : %10.1f cycles/call\n", + (double)u64AddRemove / (2.0 * N)); + Trace(" NnueForward : %10.1f cycles/call\n", + (double)u64Forward / (double)N); + if (g_fNnueProbeLoaded) + { + Trace(" NnueProbeScore : %10.1f cycles/call\n", + (double)u64Probe / (double)N); + } + return; + } + Trace("Usage: nnue load <filename>\n" + " nnue probeload <filename>\n" " nnue eval\n" " nnue on\n" - " nnue off\n"); + " nnue off\n" + " nnue lazyprobeon\n" + " nnue lazyprobeoff\n" + " nnue bench\n"); } diff --git a/src/docs/NNUE.md b/src/docs/NNUE.md index 0ab46ec..dbc5d54 100644 --- a/src/docs/NNUE.md +++ b/src/docs/NNUE.md @@ -1034,16 +1034,187 @@ interchangeable with real Stockfish `.nnue` binaries. 2,298,648 positions, 84,302 exact rows left unlabeled), `games.holdout.labeled.jsonl` (3,005 games, 255,245 positions, 9,113 exact rows left unlabeled). -- **Not yet built**: the Python training script itself (the loss - function, quantization-aware export pass), the feature-indexing-spec - cross-checker, any `SEARCHER_THREAD_CONTEXT`/accumulator plumbing - (still correctly gated behind a dedicated branch, not started), and - the lambda sweep/selection for blending `p_search`/`r_outcome`. The +- **Update 2026-10-15: all of the below is now done** (see "Status + (2026-10-15)" further down for the full writeup) -- the training + script, feature-indexing cross-checker, accumulator plumbing, and + the first trained/exported/wired-in/match-played net all exist now. + **Still not done**: the lambda sweep/selection for blending + `p_search`/`r_outcome` (still correctly deferred to Phase B, per the + lambda decision above -- no new information has arrived that would + change that), and quantization (see "Quantization plan"). The quiet-filter gap previously noted here (not yet applied to the game-shaped/labeled files) is closed -- see `quiet_filter_games.py` above -- and a future annotation run can skip the post-hoc step entirely via `annotate.py --prefilter-quiet`. +## Status (2026-10-15): training run finished, net wired into real search, first match-play result + +- **Phase A training run finished.** `annotate_pgns/nnue_train.py` against + the ~2.08M-row quiet-filtered pool above, 20 epochs, PyTorch on CPU + (per the Phase A local-CPU decision below) -- final train MSE 0.0065, + holdout MSE ~0.022. Checkpoints: `runs/sd10_twic_2M/nnue_ckpt/ + ckpt_epoch000.pt` through `ckpt_epoch019.pt`. +- **Feature-indexing cross-checker built and passing** + (`cross_check_features.py` against `nnue_feature_spec.py`'s Python + reference encoder and `src/nnuefeatures.c`'s C reference dump, 9 FENs, + all matching) -- closes milestone 1 below. +- **Weight export pass built** (`export_nnue_weights.py`): flat binary + format, 4-byte `"NNU1"` magic, `<4I` dims header + (`NNUE_DIMENSIONS`/`L1`/`L2`/`L3`), then the eight weight/bias arrays + in a fixed order. `ckpt_epoch019.pt` exported to + `runs/sd10_twic_2M/nnue_ckpt/weights.nnue` (23MB). +- **Accumulator plumbing built, tested, and proven fast** (on the + `nnue-accumulator-plumbing` branch, per milestone 3's branch + requirement) -- `SEARCHER_THREAD_CONTEXT` gained `sNnueAccum[2]`; + all 8 of `move.c`'s Lift/Place/Slide(WithoutSigs) primitives + maintain it incrementally (chose primitive-hooking over re-deriving + move-kind from `mv` after discussion -- these primitives already + couple tightly with Zobrist hashing, and `GenerateMoves`/ + `MakeMove`/`UnmakeMove` are too hot to afford re-branching); king + moves trigger a full `NnueAccumRecompute` for their *own* perspective + only (every feature in that perspective is own-king-relative), with + a `PLY_INFO`-indexed backup/restore for `UnmakeMove`. `testnnue.c`'s + `TestNnueAccumulator` plays real games from 7 curated FENs chosen to + hit captures/promotions/en passant/castling/plain king moves within + 2 plies, comparing incremental maintenance against a from-scratch + `NnueAccumRecompute` after every move -- caught two real bugs on its + first-ever runs (a king-capture backup-ordering bug where the + backup was taken after the capturing `LiftPiece` instead of before + it; and, when the perf-gating fix below landed, the test's own + failure to set `g_fUseNnueEval` before exercising the gated code). + `setboard`'s downstream effect is covered by the same + `ReInitializeSearcherContext` choke point every `go`/`ponder` already + calls -- confirmed this is the one place that needs a hook, not + `SetRootPosition` itself. +- **Load/call infrastructure and real search integration built**: + `NnueLoadWeights` (`nnueeval.c`) loads an exported `.nnue` file, + dimension-checked against the build's own constants; `NnueForward` + runs the real FC head (ReLU(accumulator) -> own-perspective-first + concat by side-to-move -> dense 2L1->L2->L3->1). `Eval()` now + short-circuits to `NnueForward` when `g_fUseNnueEval && + g_fNnueWeightsLoaded`. Reachable via the `nnue load|eval|on|off` + command and a `--nnueweights <file>` CLI flag (the latter needed + because `match_play.py` spawns engines with fixed args -- no chance + to send interactive commands first). +- **Critical performance regression found and fixed.** The accumulator + maintenance added to move.c's primitives ran unconditionally on the + (unverified, wrong) assumption that it was cheap -- cost ~50% of + engine throughput even with NNUE off (673k nps measured vs. the + expected 1.4-1.5M baseline). Gated all of it behind `g_fUseNnueEval`. + Confirmed via `bench`: classic HCE back to ~1.52M nps, NNUE eval a + fair ~410-411k nps. +- **`test_vs_head.sh` extended with `NNUE_WEIGHTS`**: wraps the + candidate binary in a thin exec shim that always injects + `--nnueweights $NNUE_WEIGHTS`, since `match_play.py`'s `Engine` class + passes identical `engine_args` to both sides and spawns the given + path directly via `Popen` (no shell), so a shebang wrapper script in + the candidate's place works unmodified -- no `match_play.py` patch + needed. +- **First real match-play result, stopped by hand at 70 games**: `-45 + =0 +25` (score 0.357), llr trending toward H0 but nowhere near + resolved (far too few games for this SPRT's bounds). At `--st 4`, + the NNUE candidate searched ~3.4-3.8 plies shallower than the HCE + head (avg depth ~10.1-10.5 vs ~13.7-13.9) and at roughly 4.6-4.8x + fewer nps (~285-293k vs ~1.32-1.41M) -- yet still won 25 of 70 games + and held a stable ~0.30-0.36 score across the run rather than + eroding toward 0 as the sample grew, and ran a *tighter* tree despite + running under HCE-tuned pruning margins (cumulative ebf + ~3.8-3.97 vs. head's ~4.59-5.01). Read as: the net's per-position + judgment looks genuinely promising, but is not yet being given a + fair fight -- two known confounds stack against it simultaneously + (raw search-depth handicap from the speed gap, and futility/ + delta-pruning/aspiration-window margins all tuned against HCE's + score distribution, not NNUE's). **Zero draws is not a red flag** -- + expected signature of a large strength/depth mismatch (the weaker + side tends to lose material in sharp lines well before either side + would reach a balanced technical ending), not evidence anything is + broken. +- **Notably, draws were exactly zero** across all 70 games -- see + above, this is explained by the depth/strength gap, not a bug. +- **Quantization investigation started** (not yet implemented -- see + "Quantization plan" below for the concrete next steps): + - AVX2 build support added (`AVX2=1` GNUmakefile option, scoped + opt-in like `GPROF=1`, not folded into the default/`-mtune=native` + set -- same portability reasoning as the existing `-march=native` + rejection). Confirmed clang's `-O3 -ffast-math` already + autovectorizes both `NnueForward` and `NnueAccumAddFeature`/ + `RemoveFeature` to SSE (4-wide) without this flag; `AVX2=1` + widens them to real 256-bit `ymm` ops (verified via `objdump` on + the linked binary, not the `.o` -- the `.o` files are LTO + bitcode, `objdump -d *.o` shows nothing meaningful). + - **AVX2 is a ~3% win for classic HCE but a measured no-op for + NNUE** (`bench`: 410,380 -> 411,402 nps, within noise). This is + a real, useful negative result: it confirms the NNUE bottleneck + is memory-bound (cache misses gathering rows out of the ~23MB + `g_rgfNnueWeight` embedding table on every accumulator + add/remove -- the per-*move* hot path, far more frequent than + the once-per-evaluated-leaf FC head), not FP throughput -- wider + SIMD can't help a stall waiting on a cache-line fetch. + - **Quantization safety-checked against the real checkpoint before + committing to any scale constants**, since this net trains with + plain (unclipped) ReLU, not Stockfish's clipped ReLU that makes + its own int16/int8 scheme near-lossless by construction. Loaded + `ckpt_epoch019.pt` directly: `ft.weight` max|.|=5.01, + std~=1.00; FC weights all max|.| < 0.36. Sampled 2000 trials of + 32 active features (a realistic full-board HalfKAv2_hm count) + summed through `ft.weight` + bias: worst-case accumulator + magnitude ~26. **Conclusion: safe to quantize this checkpoint + directly, no retrain needed** -- even a generous fixed-point + scale (256-512) leaves int16 (+-32767) comfortable headroom + against the realistic worst case (~6,600-13,300), nowhere near + saturation. + +## Quantization plan (not yet implemented) + +Priority order, chosen from the AVX2 experiment above: the embedding +table (per-*move* cost, memory-bound) matters more than the FC head +(per-evaluated-*leaf* cost, compute-bound and already partly +vectorized), even though the FC head is the easier, more isolated +change. Planned sequence: + +1. **Int16-quantize the embedding table first** (`g_rgfNnueWeight`, + `g_rgfNnueBias`) -- halves its footprint (~23MB -> ~11.5MB), + directly targets the measured memory-bound bottleneck via improved + cache hit rate on `NnueAccumAddFeature`/`RemoveFeature`'s per-move + gather. Fixed-point scale TBD from the 256-512 range checked safe + above; `NnueAccumRecompute`'s accumulation itself can likely stay + int32 (cheap, avoids overflow entirely) even once the table it + gathers from is int16. +2. **Int8-quantize the FC head second** (`nnueeval.c`'s `NnueForward`) + -- isolated to one file, runs once per evaluated leaf (not per + move), and gets the largest *relative* SIMD win (int8 dot products + are 32-wide under AVX2 vs. SSE's 4-wide float32) once it's worth + doing at all. +3. **New weight-file format version required.** `NnueLoadWeights` + currently validates magic (`"NNU1"`) + dims but has no dtype/format + field -- an int16/int8 export with the same dims would silently be + read as float32 and produce garbage, exactly the failure class the + existing dim-check exists to prevent. Bump to `"NNU2"` (or add an + explicit format word) before shipping any quantized export. +4. **New correctness test required -- `TestNnueAccumulator` will not + catch quantization error.** It only compares incremental + maintenance against `NnueAccumRecompute`; once both sides are + quantized identically they still agree with each other regardless + of whether quantization degraded the actual eval. Need a separate + check: float `NnueForward` vs. quantized `NnueForward`, same + position, across a few hundred FENs, comparing the resulting + centipawn deltas against some acceptance threshold (TBD). +5. **Side benefit, not the goal**: integer incremental accumulator + add/subtract is exact (no float drift), so quantizing removes both + `_CompareAccumToRecompute`'s 0.01f epsilon tolerance and any + long-move-sequence floating-point drift question entirely. +6. Re-run `test_vs_head.sh` with `NNUE_WEIGHTS` after each step to + measure the real nps delta before moving to the next step -- same + reasoning as checking AVX2 in isolation before quantizing: don't + stack two speed changes in one measurement. +7. Once NNUE's effective search depth is no longer handicapped by + several plies at the standard time control, re-run the + `match_play.py` gate for a real read on judgment quality, largely + decoupled from the raw-speed confound -- and only then decide + whether margin recalibration (see "Integration into search" above) + is worth doing before or after a Phase B (GPU-trained, larger net) + run. + ## Training compute - No reliable local GPU. Decided against Google Colab's notebook-session @@ -1118,36 +1289,1404 @@ interchangeable with real Stockfish `.nnue` binaries. existing joint-table instrumentation, and only build an independent bitboard pass if that measurement says it's worth it.** No speculative tactical-signals pass planned up front. -3. MakeMove/UnmakeMove accumulator plumbing (ply-indexed king-move - backup in `PLY_INFO`) built and proven correct/fast **with the old - `Eval()` still live** -- prove the plumbing before any trained net - exists to evaluate against it. **Do this on a separate git branch**, - not on `master`, given how much live search-path code it touches; - snapshot `head_reference` first. -4. TWIC/lichess annotation harness built; `<TT>`/`<EGTB>`/`<REC>` tag - semantics investigated and resolved; first large annotated training - pool assembled (Phase 1 data, per above). **Unblocked now, independent - of steps 1-3** -- no engine-internals risk, can start immediately. -5. First bootstrap net trained (rented GPU), quantization-aware export - pass built. -6. C inference code (forward pass + quantized weight loader) written and - validated against the Python model's output on held-out positions. -7. Wired into search behind a toggle; margins recalibrated. -8. `match_play.py` gate; iterate; retire old `Eval()` only after a - convincing, repeated win. +3. **Done, on `nnue-accumulator-plumbing`.** MakeMove/UnmakeMove + accumulator plumbing (ply-indexed king-move backup in `PLY_INFO`) + built and proven correct via `testnnue.c`'s `TestNnueAccumulator`, + and proven fast after finding and fixing a real ~50% throughput + regression (see "Status (2026-10-15)" above) -- gated behind + `g_fUseNnueEval` so it costs nothing with NNUE off. `head_reference` + snapshot already existed per the existing protocol. +4. **Done.** TWIC annotation harness built and run (Stage A, 2.55M + positions from 30,000 `twic_filtered.pgn` games); `<TT>`/`<EGTB>`/ + `<REC>`/`<=>`/`"only"` tag semantics investigated and resolved (see + "Status (2026-10-14)" above, including the real engine bugs found + along the way). +5. **Done for Phase A (CPU-trained bootstrap, not the GPU-trained + Phase B net).** First net trained (`nnue_train.py`, 20 epochs, + train MSE 0.0065 / holdout MSE ~0.022) and exported + (`export_nnue_weights.py` -> `weights.nnue`). **Quantization-aware + export pass is explicitly NOT done yet** -- current export is + plain float32; see "Quantization plan" above for the concrete, + not-yet-started next steps. +6. **Done**, modulo quantization. C inference code (`nnueeval.c`'s + `NnueForward` + `NnueLoadWeights`) written and running against real + search positions via the `nnue eval` command and live search + integration. **Not yet validated against the Python model's output + position-by-position** -- worth doing (a batch of FENs run through + both `nnue_model.py`'s `forward()` and `NnueForward`, comparing + scores) before trusting the C side's numerical correctness beyond + "it loads and produces plausible-looking scores," which is all + that's been checked so far. +7. **Done.** Wired into search behind `g_fUseNnueEval`/ + `g_fNnueWeightsLoaded` (`Eval()`'s short-circuit). **Margins NOT + yet recalibrated** -- still running under HCE-tuned futility/EFP/ + LMR/NMP margins, which the first match-play result's analysis + flags as a likely real confound, not yet quantified. +8. **In progress, first result in hand, inconclusive by design (stopped + early).** `match_play.py` gate: 70 games, `-45 =0 +25`, llr nowhere + near either SPRT bound. Not resumed yet -- current priority is + closing the speed gap (quantization) before spending more match + time on a comparison known to be confounded by it. Old `Eval()` + stays live and is the default (`g_fUseNnueEval` defaults `FALSE`) + until NNUE wins convincingly and consistently under this gate. + +## Status (2026-10-15, quantization session): both quantization steps done, validated, and measured + +Both steps of the "Quantization plan" above are now implemented, on +the `nnue-accumulator-plumbing` branch, in this order: + +- **Step 1 -- embedding table int16, done.** `g_rgfNnueWeight`/ + `g_rgfNnueBias` (`chess.h`/`nnueaccum.c`) are now `SHORT`, at a fixed + scale of 4096 (checked against `ft.weight`'s real max|.|=5.01 -- + `5.01*4096 ~= 20502`, well under `MAX_SHORT`, ~1.6x headroom for a + larger Phase B net). `NNUE_ACCUMULATOR.rgiValue` (was `rgfValue`) is + `INT` (int32), not `SHORT` -- summing up to 32 active features' + `SHORT` rows can exceed `SHORT`'s range even though each individual + row fits (measured worst-case real accumulator magnitude ~26 in + float-equivalent units, i.e. ~106,500 at this scale -- 3x past + `MAX_SHORT`). `NnueForward` converts back to float (divides by + `g_fNnueFtScale`) only where the accumulator is read, never where + it's maintained -- incremental `MakeMove`/`UnmakeMove` updates stay + purely additive integer arithmetic, exact, no float drift. + `testnnue.c`'s `TestNnueAccumulator` updated to match (integer test + weights, exact-equality comparison instead of an epsilon tolerance -- + a real correctness upgrade this dtype change enabled, not just a + mechanical port). Passes clean under `DEBUG=1 TEST=1`. +- **Step 2 -- FC head int8, done.** `g_rgfNnueFc1Weight`/ + `g_rgfNnueFc2Weight`/`g_rgfNnueFc3Weight` (`chess.h`/`nnueeval.c`) + are `CHAR` (int8, signed), one weight scale per layer derived + exactly from that layer's own max|.| (no sampling needed, weights + are fixed and known). Biases stay `float32` (tiny arrays, added + post-dequant, not worth quantizing). Activations are UCHAR (uint8, + unsigned -- confirmed all three stages are strictly post-ReLU, + `min=0.0000` on a real 50k-position sample, so no bits wasted on a + sign). `NnueForward` rewritten stage-by-stage: dequantize accumulator + -> ReLU -> requantize to UCHAR at AS1 -> int8x8 dot product + accumulated in `INT` -> dequantize (divide by `AS*WS`) + bias -> ReLU + -> requantize at AS2 -> repeat through fc2/fc3. Matches a Python + numpy simulation of the identical quantized math, built and checked + before touching any C code (see below). + - **Activation scales (AS1/AS2/AS3) are calibrated from real + sampled positions, not a theoretical bound** -- unlike weights, + activation ranges aren't knowable from the checkpoint alone. + `export_nnue_weights.py`'s `CalibrateActivationScales` runs + 50,000 real positions from `tensors_train/` through the float + model with forward hooks, takes the observed max per stage, and + applies a 1.3x headroom multiplier before deriving the scale + (`AS = 255 / (observed_max * 1.3)`). Real observed maxes: + `fc1_in` 27.12, `fc2_in` 19.11, `fc3_in` 7.72 (p99.9 well below + max in each case -- confirms the max is tail-driven, headroom is + doing real work, not padding noise). + - **File format bumped to `"NNU3"`** (`NnueLoadWeights` rejects + both older `"NNU1"` and `"NNU2"` files explicitly, not silently). + Header now carries `ft_scale` (1x float), `act_scale` (3x float: + AS1/AS2/AS3), `w_scale` (3x float: WS1/WS2/WS3) before the + weight arrays themselves. + - **Accuracy measured before touching any C code, same discipline + as step 1's safety check**: a numpy simulation of the exact + quantized int8 math (uint8 activations, int8 weights, int32 + accumulation, per-layer scales) against the real float model, + 5000 real holdout positions: **mean |delta| 1.55cp, median + 0.99cp, max 20.91cp, zero clipping anywhere** (weights or + activations, all three layers, out of 2.56M+160K+1K+32 values + checked). Context: the project's own `sd10`-vs-`sd16` + depth-adequacy check (training-label noise already tolerated) + measured mean |delta| 19cp -- this quantization's error is + **over 10x smaller** than noise already baked into the training + data. Confirmed again directly in C after implementation: `nnue + eval` on 4 hand-picked real FENs vs the Python float reference, + deltas of 1/2/11/0 cp -- consistent with the offline simulation, + no sign of an implementation bug. +- **AVX2+FMA confirmed as a real, separate win once real weights are + loaded** (not the "measured no-op for NNUE" result from the + pre-quantization investigation above -- that measurement predated + knowing which function actually dominated runtime, see the + profiling finding below). `gmake AVX2=1`: `objdump` on the linked + binary confirms `NnueForward`'s dot-product loops widen from 4-wide + `xmm`/`mulps`+`addps` to 8-wide `ymm` with 99 real `vfmadd*` fused + instructions. Measured: int16-table-only build went from ~381-384k + nps (no `AVX2=1`) to ~409-410k nps (`AVX2=1`) -- a real, repeatable + ~7% win, zero retrain, zero risk. +- **Profiling (`docs/PROFILE.md`'s FreeBSD `pmcstat -P` + `gprof` path) + found `NnueForward` -- the FC head, not the embedding table -- + dominates runtime: 34.0% of total self-time, alone**, more than + double the next entry (`SlidePiece`, 12.6%) and roughly on par with + *all* accumulator-maintenance-related functions combined even when + generously over-attributing their non-NNUE move-bookkeeping cost to + NNUE. `NnueAccumAddFeature`/`RemoveFeature` don't even appear as + separate flat-profile line items -- small enough to get inlined into + their `SlidePiece`-family callers. This directly revised the + quantization plan's priority ordering: the embedding table (step 1) + was assumed memory-bound-dominant from the AVX2 experiment's + "no-op for NNUE" result, but that result just meant AVX2 doesn't + help *that* code path -- it didn't establish the table was the + actual bottleneck. `fc1`'s matmul (`2*L1 x L2 = 2*256*32 = 16,384` + multiply-adds per evaluated leaf) being far larger than `fc2` + (1,024) or `fc3` (32) explains why: cost at each layer boundary + scales with the *product* of adjacent layer widths, and the + 512-wide-to-32 transition is the one expensive step in this + topology, more frequently hit (once per evaluated leaf) and more + expensive per-call than the table gather (once per move, but only + a handful of 256-element row reads). +- **Net bench result, chaining every change in this session**: + + | Build state | nps (real weights, `nnue on`) | + |---|---| + | Float32 everywhere (pre-session baseline) | ~411,000 | + | int16 table, no `AVX2=1` | ~381,000-384,000 | + | int16 table + `AVX2=1` | ~409,000-410,000 | + | int16 table + int8 FC head + `AVX2=1` | **~559,000-563,000** | + + Net effect: **~37% faster** than the pre-session float32 baseline, + closing the gap to classic HCE (~1.51M nps) from 3.9x down to 2.7x. + Correctness verified at every step (self-tests, offline Python + simulations before touching C, then direct C-vs-Python spot checks + after). +- **`WS1`/`WS2`/`WS3` derivation note**: unlike the embedding table + (one global scale, since it's one homogeneous array), each FC layer + gets its *own* weight scale, derived from that layer's own max|.| -- + `fc1.weight` max 0.350, `fc2.weight` max 0.361, `fc3.weight` max + 0.334 (all much smaller than `ft.weight`'s 5.01, but the relevant + comparison for int8 resolution isn't the max, it's the *step size* + relative to each layer's std -- at full-range scale this works out + to roughly 18 quantization levels per standard deviation for the FC + weights, versus thousands for the int16 table. The 1.55cp measured + result shows this coarser per-weight resolution doesn't translate + 1:1 into coarser *output* accuracy -- dot-product summation over + many terms (512/32/32) partially cancels independent rounding + errors, and ReLU's sparsity (46-69% exact zeros measured on real + activations) contributes zero error for free on a large fraction of + terms. + +## Lazy-evaluation tiering plan (started 2026-10-15, not yet wired into search) + +Classic `Eval()` has three real exits -- super-lazy (material only), +lazy (material + a defined subset of positional terms), full -- each +gated by a margin derived from a *measured population* of real +positions, not a modeled/assumed distribution. `Eval()`'s current NNUE +short-circuit (added when NNUE was first wired into search, see +"Status (2026-10-15): training run finished" above) bypasses all of +this -- it's an unconditional full `NnueForward()` call, with a comment +explicitly noting the margin machinery below it "has not been +recalibrated against NNUE's score distribution yet." This section +plans closing that gap. + +**The mechanism, read directly out of the live code** (not +reconstructed from memory -- see `eval.c`'s +`iSuperLazyResidualPessimisticByOwnArmy`/`OptimisticByOwnArmy`/ +`P50ByOwnArmy` tables and their use in `Eval()`'s super-lazy exit, +~line 7150): + +- Every one of these tables is **literally empirical percentiles read + off a measured population of real positions** -- the comment blocks + above several of them show raw population stats before the table + (e.g. `n=104187433 min=-68 p1=-10 ... p50=10 ... max=165`). No + distribution shape is assumed; the percentile is read directly off + sorted real data, per material bucket (`_MaterialBucket`/per-side + army-scaler bucket, 8 buckets in the existing tables). +- **Two different statistics for two different jobs, not one margin**: + - **p50 is a bias correction** -- the typical/unbiased value fed to + `iPositional` (a lazy exit's best-guess estimate of the missing + positional terms, consumed downstream by e.g. QSearch's own + futility margins). Current NNUE short-circuit reports a flat + `{0, 0}` here -- the "intentionally least-specific but safe" + placeholder, not a real estimate. A calibrated p50 of the probe's + own residual could improve this once real residual data exists. + - **p2/p98 (not p5/p95) is the soundness bound** -- answers "how far + could the true full-eval score plausibly be from this cheap + estimate, in the realistic tail," and is what actually gates + whether the lazy exit is safe to take. The exact comment: *"p50 is + the right statistic for a bias correction, p2/p98 is the right + statistic for a soundness bound."* + - Direction is asymmetric and combines two terms before a `MAX0()` + floor: checking against `alpha` (could this fail low?) uses the + **optimistic** tail (most generous plausible correction in the + mover's favor); checking against `beta` (could this fail high?) + uses the **pessimistic** tail (least favorable plausible + correction). Getting this sign/direction wrong would silently + invert the safety guarantee -- needs care, not pattern-matching, + when implemented for real. +- This two-statistic pattern (p50 bias, p2/p98 safety bound) is what + the NNUE lazy tier should reuse, not a new invention -- same + methodology the existing calibration harness + (`CALIBRATE_MARGIN_SAFETY`, `RecordSuperLazyMarginSafetySwing`, swing + histograms bucketed by material) already proved out for classic + `Eval()`'s own tiers. + +**The proposed NNUE lazy-tier signal**: not a second, separately +trained net (that reintroduces the calibration-mismatch problem +already rejected in the "Architecture decisions" section above) -- +instead, a **linear probe fit directly on the real net's own +accumulator** (`fc1`'s input: the ReLU'd, concatenated 512-wide +accumulator halves), which is already computed and resident at every +node for free as a side effect of `nnueaccum.c`'s incremental +maintenance, before `NnueForward`'s expensive `fc1->fc2->fc3` pass +(34% of runtime, see profiling finding above) ever runs. + +**Progress so far**: +- **Offline probe fit, done, preliminary.** Ridge regression (512 + features + bias, lambda=1.0) fit against 80,000 real positions from + `tensors_train/`, targeting the real float net's own raw output + (`forward()`, pre-sigmoid). Saved to + `annotate_pgns/probe_weight.npy`/`probe_bias.npy`. +- **Measured residual (full net score minus probe score), 20,000 held-out + positions from the same sample**: mean |residual| 50.10cp, median + 33.09cp, p2=-166.79cp, p50=0.33cp, p98=+169.14cp. **An order of + magnitude coarser than the int8 quantization's 1.55cp** -- expected, + since a flat linear projection discards all of `fc1->fc2->fc3`'s + nonlinear structure, unlike int8 which preserves the same + computation at lower numeric precision. This sets real expectations: + the implied safety margin (~150-200cp) is large relative to typical + eval swings, so the lazy exit will only fire on nodes already + lopsided relative to the search window -- a real but likely narrower + slice of total nodes than hoped going in. +- **Real fit-vs-validation gap observed** (mean 37.85cp on the fit + split vs 50.10cp held out, p98 153cp vs 195cp) -- larger than + expected given 80k examples against 512 features plus ridge + regularization. Not yet root-caused (could be regularization + strength not cross-validated, could be real limits of a flat linear + fit over this representation, could be this offline sample's size). + **Flagged as unresolved, not yet trusted as final** -- this was an + offline, single-position-sample check, explicitly a preliminary + step, not the real calibration. + +**Not yet done -- the actual next steps, in order**: +1. Root-cause the fit/validation gap above (cross-validate `lambda`, + or just use more fitting data) before trusting any residual + numbers from this probe as final. +2. **Build the real calibration pass**: a `CALIBRATE_MARGIN_SAFETY`- + style harness for the NNUE lazy probe specifically -- same pattern + as classic `Eval()`'s existing one (run the full `NnueForward` pass + anyway even when the probe-based exit would fire, record the swing, + restore the real build's behavior before returning) -- against + **real search-node traffic**, not another offline position sample. + This is the measurement that actually matters; the offline check + above was only ever a cheap sanity gate before investing in this. +3. From that real population, derive p50 (bias correction) and p2/p98 + (safety bound) -- possibly bucketed by material/phase the way + HCE's own tables are, possibly not. **Whether NNUE's accumulator + already implicitly encodes phase** (unlike HCE's explicitly + material-gated lazy terms) **is an open, empirically-answerable + question from this same population** -- check whether the residual + distribution actually shifts meaningfully bucket-to-bucket before + committing to that complexity. +4. Wire the gate into `Eval()`'s existing NNUE short-circuit, using + `iAlpha`/`iBeta` already passed into `Eval()` -- no new interface + needed, this is a parallel early-return block next to the current + unconditional bypass. +5. Ship it instrumented-only first (swing recorded, real build + behavior unaffected) before trusting it live, same rollout + discipline as every other change in this project. + +The super-lazy (material-only, no net at all) idea from the +"Architecture decisions" section above remains separate, more +tentative, and explicitly not decided either way -- the calibration +risk there (no shared units with the WDL-sigmoid-calibrated net +output) is structurally different from this linear-probe tier, which +shares the real net's own trained representation and units by +construction. + +## Status (2026-10-16): lazy probe gate shipped live, QSearch futility margins fixed, same-depth ECM win confirmed, overnight match queued + +Picking up from the "Lazy-evaluation tiering plan" section above -- +the fit-vs-validation gap was root-caused as pure sampling noise (more +data + the real `tensors_holdout/` split made it vanish; lambda barely +mattered, the model was never near overfitting), and everything from +"build the real calibration pass" onward is now done. + +- **Fit/validation gap resolved**: 500k-example fit + real `tensors_ + holdout/` evaluation (not another ad-hoc split) showed holdout + *better* than fit (38.68cp vs 41.09cp mean) -- the earlier 50.10cp + number was noise from a too-small single sample, not overfitting. +- **Real calibration done properly, twice**: first pass used the three + curated tactical suites (91M samples) -- too narrow a slice (hand- + picked for sharp tactics), confirmed by a second pass against + `tests/cross_sectional_200k.ep_` (200,000 diverse positions, 16 + parallel single-threaded shards, `sd 6`, 3.09 BILLION real search- + node samples) landing close to but meaningfully more trustworthy + than the first (p98 margins `[180, 200, 190, 180, 170, 160, 150, + 150]` by material bucket vs the tactical-suite-only `[190, 200, 200, + 190, 190, 170, 150, 150]` -- also surfaced real tail outliers (max + 534cp) the narrower sample never saw). +- **A real, separate bug found and fixed along the way**: chaining + multiple commands after `script <file>` in one `--batch --command` + string (e.g. `script a.ep_; script b.ep_; calibrate marginsafety`) + silently truncates every search except the last to a shallower depth + than requested -- `PushNewInput` (`input.c`) splits and queues + *everything* up front, and `searchsup.c`'s mid-search + `NumberOfPendingInputEvents()` check (legitimate for live xboard/ + WinBoard responsiveness) fires on that queued-but-irrelevant trailing + text, treating it as "stop, something wants my attention." Root + cause confirmed by direct repro (`sd 10` silently landing at depth 8, + 8x fewer nodes) -- **not fixed in the engine** (the interactive- + responsiveness behavior is legitimate and deliberately left alone; + the input-queueing semantics are too sensitive to touch for a + testing-only problem that already has a zero-risk workaround). + **Workaround, now the required pattern for any multi-step batch + calibration**: never chain unrelated commands after `script` in one + `--command` string; if a command needs to run after a suite finishes + in the same process (e.g. `calibrate marginsafety`), put it as its + own line *inside* the `.ep_` file instead -- `script.c`'s own + `_ProcessSuiteLine` already dispatches any non-"go" line via the + queue-free `ExecuteCommandLineDirectly`, so it runs in-order with no + truncation risk. For N-way sharded runs (per CLAUDE.md's existing + convention), each shard gets its own single `script` call + its own + `calibrate marginsafety <path>` line inside the file, in its own + process -- then combine the N raw histogram files after the fact. +- **The "super lazy" (material-only) tier was investigated and + rejected as a live feature** -- `pos->iMaterialBalance[color]` is + already in the same cp units as `NnueForward`'s output + (`VALUE_PAWN=100`), so no fit/regression was needed at all (unlike + the accumulator probe) -- straight subtraction, same harness. Real + measured residual against the full net, same 3.09B-sample + cross-sectional pool: mean 124-344cp, p98 510-1230cp by bucket -- + **6-8x larger than the accumulator probe's** (47-77cp mean, 150-200cp + p98). Shape is an inverted-U by material bucket (peaks in the + middle, smaller at both bare-king endgames and near-full-material + positions), not the simple monotonic-opposite-of-the-probe trend + originally guessed. Conclusion: material alone is too crude a proxy + for *this* net's output to fire often enough to be worth it -- NNUE's + accumulator captures far more positional judgment relative to + material than a handcrafted eval does, unlike HCE where material + really is close to the final answer. Not built as a live gate. +- **Lazy probe gate shipped live** (`nnue lazyprobeon`/`lazyprobeoff`, + defaults `FALSE` same as `g_fUseNnueEval`): `Eval()`'s NNUE short- + circuit now actually skips `NnueForward` when + `NnueProbeScore() +/- margin` confidently resolves fail-high/fail-low + against `[iAlpha, iBeta]`, dual-mode via `#ifdef CALIBRATE_MARGIN_ + SAFETY` (calibration build always still runs the full pass for + measurement; release build actually skips). Held-out validation + (`ecm_confident_sample2.ep_`, not used for calibration) showed + exceeded rates mostly at or under the ~2% the p98 choice implies + (buckets 2-7: 0.87-2.24%); buckets 0-1 ran hotter (3.47-4.67%, + consistent with those being the smallest-sample buckets in + calibration, so the noisiest estimates). +- **Measured lazy-probe exit rate on real search traffic** (`twic_ + sample.ep_`, `sd 6`): **14-23% of NNUE eval calls resolve via the + cheap probe**, varying by material bucket (peak ~20-23% in buckets + 3-5, low 8.7% in bucket 7/near-full-material). Most calls still pay + for the full forward pass. +- **Real, clean, sequential (not parallel -- see below) same-depth + measurement, `ecm_hard_quick`, `sd 10`, gate off vs on**: + + | | OFF | ON | + |---|---|---| + | Solved | 32/90 | 33/90 | + | Nodes | 47,790,608 | 50,438,386 | + | Time | 112.4s | 107.1s | + | nps | 425,271 | 470,938 | + + **~10.7% real per-node throughput win, no solve-rate regression** + (if anything +1). Note nodes went *up* while time went *down* -- + the gate changes which nodes get pruned (approximate score at some + nodes shifts alpha-beta cutoffs), so node count isn't held constant + the way `sd` depth is; nps is the cleaner isolated signal. + **Caveat on parallel-run timing noise, confirmed twice this + session**: running gate-off and gate-on as simultaneous background + jobs on a shared box can be contaminated by real external load (cron + jobs, rsyncs, other users) even when each job's own `--cpus 1` leaves + plenty of idle capacity -- the fix was a clean *sequential* run on an + otherwise-idle box, not avoiding parallelism itself (6-way `--cpus 1` + parallel jobs were confirmed fine once the box was actually quiet). +- **Full three-suite combined result, same clean methodology**: + 108/191 (56.5%) solved OFF vs 112/191 (58.6%) ON, total time 247.7s + vs 240.3s (-3.0%) -- consistent with the single-suite sequential + number, no regression anywhere. +- **Independently, at the SAME fixed `sd 10` depth (no lazy gate, + separate from all of the above): NNUE solves substantially more ECM + tactics than classic HCE** -- 108/191 (56.5%) vs HCE's 72/191 + (37.7%) across all three curated suites. Since depth is held + constant, this isn't explained by NNUE's remaining speed deficit. + **Correction, 2026-10-17: do not read this as evidence the net + plays better chess.** Every real head-to-head game match run + since (see "Status (2026-10-17)" below -- five independent + configurations: two different nets, `st 4` and `sd 10`, with and + without the lazy probe) has NNUE losing decisively to classic HCE, + including at matched search depth where this ECM result would + predict an NNUE edge. ECM is a narrow slice (forced-tactic/mate + puzzles, where qsearch/extensions/move-ordering machinery shared + by both evaluators does most of the work) and evidently does not + generalize to real-game strength. As of this writing, solving more + ECM puzzles than HCE is the *only* thing either NNUE net has been + shown to do better than HCE -- it has not won a single head-to-head + match, at any depth or time control tried. +- **QSearch futility-pruning margins needed their own investigation, + separate from the eval-exit-gate margins above** -- `PLY_INFO. + uLastPositionalSource` (the per-ply tag `search.c`'s + `FUTILITY_BASE_MARGIN_BY_SOURCE[]` reads to decide how much slack to + give QSearch's delta/futility pruning) was **never being set at all** + by either NNUE exit -- every NNUE-sourced score was silently + inheriting a stale value from the last classic-`Eval()` call at that + ply slot (or the C zero-init default, `EVAL_POSITIONAL_SOURCE_FULL`, + if none ever ran). Fixed: added `EVAL_POSITIONAL_SOURCE_NNUE_FULL`/ + `NNUE_LAZY`, set at both NNUE return sites, each with its own + `FUTILITY_BASE_MARGIN` entry and always-on exit-rate counters + (`calibrate exitrate` extended to report NNUE's own lazy-vs-full + split, separate from the eval-exit gate's). + - **First real measurement** (`twic_sample.ep_`, 1500 positions, + `sd 6`, the same pool/depth the *original* classic margins were + derived from): `nnue full`'s surprise rate was 7.41% in the + checking-capture/promo gate at the 450 starting placeholder -- + well above anything in classic's documented history. Widening to + 650 tested as a complete no-op (identical 7.41%, identical `n`). + - **Root-caused, not just observed**: added a direct diagnostic + (raw pre-clamp `iFutility` value, before `MAX0`) and confirmed + `nnue full`'s population is **100.00% clamped to the `MAX0` + floor** (mean pre-clamp -589.3) -- the margin constant is never + actually read for this population, so no value could have shown + an effect. Classic `full eval`, measured the identical way on + the identical pool, is clamped 91.05% of the time (not 100%, + and less deeply negative: mean -282.5) -- saturation is a real, + general QSearch structural property (not NNUE-specific: classic + `regular lazy`'s 24.95% clamp rate matches `nnue lazy probe`'s + 24.76% almost exactly), but NNUE's `FULL` tier saturates somewhat + more completely than classic's. + - **Surprise magnitude (not just rate) measured too**, resolving + a live "is NNUE just noisier" vs "does NNUE miss bigger things" + question: `nnue full`'s surprises, when they happen, run + *smaller* than classic `full eval`'s (mean 477.5cp vs 938.7cp, + `>100cp` 9.88% vs 28.74%) -- same pattern for `nnue lazy probe` + vs classic `regular lazy` (5085.3cp vs 11030.2cp mean; 19.45% vs + 69.73% `>100cp`). So the net effect is: NNUE_FULL fails *more + often* but *less expensively* each time -- consistent with more + frequent small eval jitter near the decision boundary, not with + NNUE being tactically blind to bigger things than HCE already + is. One sample verified by hand (a genuine `Qh7+` mate-in-1, + confirmed independently via a real `sd 16` search) to make sure + the diagnostic's rare huge-magnitude entries were real finds, not + a measurement bug. + - **Decision: `FUTILITY_BASE_MARGIN_NNUE_FULL` reverted to 450** + (matching classic `FULL`) rather than kept at the untested 650 -- + the saturation finding means neither value was actually + validated by this data, and 450 is the more defensible default + given NNUE_FULL's measured surprises skew smaller than HCE's own. + **`FUTILITY_BASE_MARGIN_NNUE_LAZY=200` kept** -- this one *is* + genuinely validated (only 24.76% clamped, moved the expected + direction when tightened from 350, landed in a sane 1.3-2.1% + surprise-rate range, smaller surprise magnitudes than classic's + own `regular lazy`). + - **TODO, not yet done**: find and calibrate against the actual + non-saturated `NNUE_FULL` population specifically (the ~0% slice + this test pool never surfaced) before trusting any `NNUE_FULL` + margin as real, NNUE-specific-validated rather than a reasonable + borrowed default. Likely candidate: nodes where `iEval` sits + close to `iAlpha` rather than comfortably above it (the + saturated case is "`iEval` way above `iAlpha`" -- look at the + complement). +- **Match-play CLI wiring added, needed before any NNUE+lazy-gate + match can run at all**: `--nnueweights` already loaded weights and + set `g_fUseNnueEval=TRUE` at startup (for `match_play.py`'s fresh- + process-per-game spawning, no interactive commands available) -- + but there was no equivalent for the probe/lazy gate at all. Added + `--nnueprobeweights <file>` (same convention, sets + `g_fUseNnueLazyProbe=TRUE` automatically), and extended `eval_tune/ + test_vs_head.sh`'s `NNUE_WEIGHTS` wrapper with a new + `NNUE_PROBE_WEIGHTS` env var that injects `--nnueprobeweights` into + the same shim. Verified working end-to-end (both flags load and + enable correctly) before trusting it for an unattended run. + **Gotcha**: `test_vs_head.sh`'s default (no candidate path given) + rebuilds the candidate itself via `gmake clean && gmake release + MP=1 PERF_COUNTERS=1` -- **without `AVX2=1`** -- silently losing the + ~7% AVX2/FMA win if not passed a pre-built candidate path instead. +- **Overnight `match_play.py` run queued** (not yet run/reported) -- + NNUE+lazy-gate candidate vs classic HCE `head_reference`, using the + new `NNUE_WEIGHTS`/`NNUE_PROBE_WEIGHTS` env vars and an explicit + pre-built `AVX2=1` candidate binary to avoid the gotcha above. + +## Next up + +1. Read the overnight match result once it's in -- first real + judgment-quality read with both the quantization work and the lazy + probe gate live together, at a real time control rather than fixed + `sd` depth (so the remaining ~2.7-3.2x speed gap is a live confound + again, same as the original 70-game result, but now with a smaller + gap and the added lazy-gate throughput). +2. The `NNUE_FULL` futility-margin TODO above -- find the non-saturated + population and calibrate against it specifically, rather than + continuing to rely on a borrowed classic-HCE default. +3. Decide whether the ~3-11% nps win from the lazy probe gate + (measured) is worth the complexity on its own, or whether it's more + valuable combined with further quantization work on the FC head's + remaining float32 pieces (biases) or other speed levers -- revisit + once the overnight match gives a real strength read to weigh + against. + +## Status (2026-10-17): material-blindness fixed and verified, but zero +## head-to-head wins against classic HCE in five independent trials + +**Summary, stated plainly up front**: a real, measured defect (the net +could not distinguish losing a pawn from losing a queen in crowded +midgame positions) was found and fixed this session. The fix is +verified to work, at the population level, with a new reusable tool. +But **no NNUE configuration tried today -- old net, new net, with or +without the lazy probe, at `st 4` or at matched `sd 10` depth -- won +or even drew competitively against classic HCE.** The only thing +either net has been shown to do better than HCE is solve more ECM +puzzles at matched depth (see the correction just above). Do not +read anything below as evidence the net plays winning chess yet. + +### The material-blindness bug, found and fixed + +- **Found via `sanity_check_net.py`** (new tool, `annotate_pgns/`): + loads a raw training checkpoint directly (no export/C round-trip + needed -- the earliest point in the pipeline a net can be checked) + and removes one piece at a time from hand-picked and (better) real + sampled positions, reporting the score delta. On the Phase A net + trained with plain `λ=1.0` MSE against WDL-sigmoid targets: removing + a pawn, a knight, a rook, or a queen from the same midgame position + produced nearly indistinguishable score drops (-0.56 to -0.70) -- + directionally correct, magnitude-blind. +- **Root cause, two compounding mechanisms, neither one specific to + which corpus a position came from**: + 1. Sigmoid-gradient argument (later found incomplete on its own -- + see below): training loss is MSE against `sigmoid(raw/K)`, and + sigmoid's gradient vanishes once a position is already clearly + lost/won. + 2. **Feature dilution / shortcut learning in crowded positions** + (the more complete explanation): with ~20+ active embedding + features summed per position, a net can match training labels + well without correctly attributing any single piece's marginal + value, because real games never present a controlled + single-piece-ablation pair to force that attribution. Confirmed + empirically: the bug was severe in crowded midgame positions and + essentially absent in sparse endgames (2-6 pieces), where each + piece dominates a much larger share of the accumulator. +- **Data rebalancing (TWIC downsampled to match self-play 1:1 by game + count) was necessary but not sufficient.** It measurably helped the + sparse-endgame case (which TWIC's human-resignation bias + structurally starves) but did nothing for the crowded-midgame case, + confirming the bug was a representation/optimization problem, not + purely a data-coverage gap. +- **Two cheap levers tried and only partially effective on their + own**: + - `loss_exponent=2.6` (Stockfish's own documented value, reweights + the loss toward large errors): modest effect, spread narrowed but + monotonicity still failed, bishop/rook order still inconsistent + across test positions. + - λ-sweep (blend in `r_outcome`, game result): **made things + actively worse** -- λ=0.5/0.0 flipped the sign of several + deltas (removing a piece appeared to *help* the score). See the + "λ-sweep methodology was weaker than later standards" note below + before trusting this conclusion, though -- it has not been + properly re-tested since. + - **Lesson from this part of the investigation**: a single + hand-picked test position cannot reliably settle a question like + "does the net value a bishop more than a rook" -- one test showed + bishop > rook, a different hand-picked position showed the + reverse. This is per-position confounding (an active bishop vs. + a boxed-in rook on that specific square/position), not + necessarily evidence about the net's general prior. +- **Fix that worked: an auxiliary material-probe loss** + (`nnue_model.py`'s `material_probe`, a plain `Linear(512, 1)` + reading the same concatenated accumulator `fc1` does). Trained + jointly with the main WDL loss: `loss = wdl_loss + + material_loss_weight * mse(material_probe(accumulator), + material_balance)`, where `material_balance` is computed directly + from the FEN (pawn=1..queen=9, mover-relative, no sigmoid -- + deliberately non-saturating). `material_loss_weight=0.01` chosen to + keep the material term's raw contribution smaller than the WDL + loss's throughout training (per the user's explicit concern about + over-fixating on material, HCE-style) -- confirmed: material loss + contributed ~0.019 vs. WDL's ~0.044 at epoch 0, ~0.007 vs. ~0.022 at + epoch 19. + - **Mechanistically why this is safer than adding material to the + output directly**: the probe loss only updates `material_probe` + itself and (via backprop) the shared `ft` embedding table -- it + never touches `fc1`/`fc2`/`fc3`, the actual eval head. The eval + head stays free to deviate from pure material counting for + positional reasons; only the *representation underneath it* is + forced to keep material linearly recoverable. + - Needed more epochs than the baseline run to show its full effect + -- material-probe MSE was still falling at epoch 19 (not + plateaued), extending to epoch 34 took it from a single hand-picked + test's queen delta of -0.815 (already beating rook's -0.727) to + a much larger, cleaner separation. + +### `sanity_check_net.py --cross-sectional`: the real validation tool + +Single hand-picked positions were repeatedly shown to be unreliable +(see above) -- replaced with a population-level check: sample N real +positions from a holdout pool, remove one random piece of each +present type per position, pool deltas by piece type across the whole +sample, report mean/median/std/p25/p75. On the matprobe net +(2000 sampled holdout positions): **population means came out cleanly +monotonic in piece value, both directions** -- pawn -0.747/+0.786, +knight -1.363/+1.581, bishop -1.500/+1.710, rook -2.483/+2.907, queen +-4.160/+5.024 (own-piece-removed / opponent-piece-removed). The +earlier bishop-vs-rook inconsistency vanished entirely at the +population level, confirming it was per-position noise, not a real +defect. Scaling is compressed relative to nominal chess piece values +(queen/pawn ratio ~5.6x here vs. nominal 9x) -- expected and likely +correct, not a bug, since training targets are WDL-sigmoid-calibrated +scores, not raw material counts, and a large material edge's marginal +effect on win probability is genuinely smaller than its point value. + +This tool (`annotate_pgns/sanity_check_net.py`) should be run on any +future checkpoint before trusting a single hand-picked-position read +-- `--cross-sectional --n-positions 2000` against a holdout pool, +`python3 sanity_check_net.py --ckpt <path>` for the quick hand-picked +smoke test. + +### Export and C-side validation: clean + +`export_nnue_weights.py` (already quantizing directly to the `NNU3` +format, no separate float32-only export step in the current pipeline) +ran against the matprobe checkpoint with zero code changes needed -- +it only reads `ft`/`fc1`/`fc2`/`fc3` by name, so the auxiliary +`material_probe` layer is silently and correctly ignored at export +time. 6 real holdout FENs spot-checked, Python float vs. C +int16/int8-quantized: deltas of 0.3-4.7cp, same ballpark as the +1.55cp-mean/20.91cp-max already measured and accepted for the first +net's quantization step. No sign/implementation bug found. + +### Five independent head-to-head trials, all losses + +All via `eval_tune/test_vs_head.sh` with `NNUE_WEIGHTS` set (and, for +two of the five, `NNUE_PROBE_WEIGHTS` pointing at the OLD net's lazy +probe -- see the stale-probe caveat below), against the classic-HCE +`head_reference`-equivalent head binary: + +| # | net | time control | result (at stop) | llr | candidate depth vs head | +|---|---|---|---|---|---| +| 1 | matprobe (new) | `st 4` | -27 =2 +1 (n=30), score 0.067 | -2.07 | 12.16 vs 15.08 | +| 2 | v3 (old) | `st 4` | -29 =1 +0 (n=30), score 0.017 | -13.05 | 10.74 vs 13.51 | +| 3 | v3 (old) | `sd 10` | -28 =2 +0 (n=30), score 0.067 | -1.52 | 9.57 vs 9.23 | +| 4 | matprobe (new) | `sd 10` | -24 =4 +2 (n=30), score 0.133 | -0.97 | 9.81 vs 9.70 | +| 5 | v3 (old) + lazy probe | `sd 10` | -30 =0 +0 (n=30), score 0.000 | (total wipeout) | 9.63 vs 9.25 | + +**Key findings**: +- **The new (matprobe) net consistently loses less badly than the old + net, under every condition tested** (trial 1 vs 2 at `st 4`: score + 0.067 vs 0.017; trial 4 vs 3 at `sd 10`: score 0.133 vs 0.067). The + material fix is a real, reproducible improvement -- just nowhere + near enough to close the gap with HCE. +- **At matched `sd 10` depth (trials 3 and 4), NNUE still loses + decisively** -- this directly contradicts the working hypothesis + (built from the ECM result above) that the real-time losses were + purely a depth/speed-handicap artifact. They are not purely that. + Matching the final depth number does not mean matching search + quality throughout the tree -- EFP/LMR/NMP/futility margins are + still HCE-tuned in every one of these trials (see "Integration into + search" above, still not done) and could still be causing bad + cutoffs specific to NNUE's score distribution even when the + reported depth matches. +- **The lazy probe used in trial 5 is STALE and the result should be + discounted**: `annotate_pgns/nnue_probe.bin` was fit against the OLD + net's (v3) accumulator specifically, before this session's retrain. + It happens to be the correct vintage for trial 5 (v3 + v3's own + probe), so that comparison is internally consistent, but no probe + exists yet for the matprobe net -- `typhoon_nnue`'s wrapper script + was updated this session to point at the matprobe net by default + and to NOT fall back to the stale probe file (previously silent and + wrong). A new probe would need to be fit + calibrated from scratch + against the matprobe net's accumulator before it could be used in a + trial 4-equivalent run. +- **A user-remembered "NNUE competitive or better" result could not + be reproduced under any of these five conditions.** Likely + explanations, unconfirmed: a different match configuration entirely + (not `st 4` or `sd 10` against this head binary), or a misremembered + result. Do not plan further work assuming that result is real until + it can be reproduced under a specific, documented configuration. + +### Pipeline sanity check: ruled out a structural bug + +Given five straight losses, including at matched depth where ECM +predicted an edge, the obvious question was "is the training pipeline +itself broken" (bad feature indexing under real gradient flow, broken +accumulator math, a training-loop bug) rather than a genuine +net-quality gap. Tested directly (`annotate_pgns/ +overfit_sanity_check.py`, new tool): sampled 3000 real positions, +scored each with the engine's own classic `Eval()` (fast +persistent-context `eval` command), and trained the *same* +architecture to directly regress raw output against that score (plain +MSE, no sigmoid, no K, no λ, no quiet filter, no quantization -- the +simplest possible test of "can this architecture memorize a small, +clean, noiseless target at all"). **Result: clean overfit** -- RMSE +collapsed from 307cp to ~3-5cp over 400 epochs on the fixed 3000-point +set (max error 46cp). **No structural bug found.** The five +head-to-head losses are real evidence about net/training quality, not +a pipeline defect. + +### Revised understanding of why: likely bootstrapping off HCE's own +### search sets a ceiling, not (as earlier argued) label noise or +### architecture capacity + +Two explanations floated earlier in this investigation don't hold up +well under scrutiny: +- **Architecture capacity**: the 256x2-32-32-1 shape is the literal + one Stockfish's own early, successful NNUE work used. Not a good + suspect. +- **"Search-score labels are noisy"**: standard practice (`sd 8`-ish + search-score training is exactly what real NNUE projects use); the + depth/source of search isn't inherently suspect. + +**The sharper version of the concern**: every label in the current +training pool -- TWIC and self-play both -- comes from THIS SAME +engine's own classic `Eval()` run through its own `sd 10` search, and +training used `λ=1.0` (pure `p_search`, zero weight on actual game +`r_outcome`). A net trained to perfectly reproduce "HCE's static eval, +distilled to one forward pass" can at best approach HCE's *positional +judgment* -- it has no training signal that could let it exceed HCE's +actual playing strength, which is that judgment plus a mature, +decades-tuned full search. This is single-generation distillation +from the exact opponent being benchmarked against, with no +independent ground truth in the loop. Real NNUE bootstrapping success +stories (including Stockfish's own) iterate through many generations, +each net's own self-play becoming the next round's data, with game +outcomes mattering throughout -- not one shot of pure +distillation-from-one-engine's-shallow-search. + +**This reframes the earlier-rejected λ-sweep as an open question +again, not a settled one** (see the "λ-sweep methodology was weaker +than later standards" note above) -- a properly-controlled λ=0.5 run +(combined with the material-probe fix, trained to real convergence, +judged with the cross-sectional population check, not a single +hand-picked position) is in progress as of this writing to answer +this properly. + +### Next up + +1. **In progress**: λ=0.5 + material-probe-loss combined run, trained + to convergence, judged with `sanity_check_net.py --cross-sectional` + -- the properly-controlled version of the λ-sweep question above. +2. Once a net shows a real head-to-head result against HCE (win, draw, + or a meaningfully-less-bad loss than trial 4's 0.133), THEN invest + in margin recalibration (EFP/LMR/NMP/futility, still 100% HCE-tuned + today) and a new lazy-probe fit for whichever net is being kept -- + not before, per this session's experience chasing search-side + explanations that turned out to not be the dominant factor. +3. Data volume scale-up (TWIC + self-play, more self-play generations + specifically using the matprobe-or-later net to play, per the + original Phase 2 plan) remains the most clearly-actionable lever, + but should be paired with an actual outcome-grounding signal in the + training target, not just more HCE-sd10-labeled positions of the + same shape already in hand. + +## Status (2026-10-18): Stockfish-labeled relabel in progress; feature +## factorization planned as the next architecture-side lever + +Labels are being regenerated using Stockfish (not this engine's own +`Eval()`/`sd 10`) to break the single-generation self-distillation +problem described above -- that's a speed/ground-truth fix, not +believed to be the root cause of the five head-to-head losses by +itself. In parallel, researching Stockfish's own NNUE docs surfaced a +second, independent candidate explanation worth trying once there's +enough Stockfish-labeled data to re-run training: **feature +factorization** (Stockfish's term for a training-only "virtual +feature set" that is coalesced into the real weights post-training; +see https://github.com/official-stockfish/nnue-pytorch/blob/master/docs/nnue.md#feature-factorization, +excerpted into this conversation). + +### Why this looks like a real lever here, not just a ported idea + +The current feature set is HalfKAv2_hm per the canonical spec above: +22528-dim, king-bucketed (32 buckets via `king_bucket()`'s +horizontal-mirror quartet), so every `(piece_square, piece_type, +piece_color)` combination exists as up to 32 separate rows, one per +king bucket, each seen only when the king happens to sit in that +bucket. Checked directly against the current real training cache +(`runs/sd10_twic_2M/tensors_train`, 1,848,411 positions): + +- **23.9% of the 22528 HalfKAv2 rows have zero observations** -- + never fire once in training, never receive a gradient update. +- **60.5% have fewer than 50 observations.** +- Collapsing the same data to the king-bucket-independent "P" feature + set (704 rows: `piece_square x piece_type x piece_color`, no king + info) gives a median of **8375** observations per row -- roughly + 32x denser, as expected, with none left permanently untrained. + +So a substantial fraction of the net's weight space is currently +stuck at random init or barely trained regardless of label quality -- +plausibly part of why two different training attempts "worked" +(loss went down, no crash) but neither net could beat HCE in +head-to-head play. This is Stockfish's own "dusty corners" framing: +sparsity from king-conditioning starves rare (king, piece, square) +combinations of signal, which factorization is specifically designed +to fix during early training. + +### Planned implementation (not yet done -- next after enough +### Stockfish-labeled data exists to re-run training) + +1. **No change to `build_training_tensors.py` / no cache rebuild + needed.** The HalfKAv2_hm index formula is + `idx = (sq ^ mask) + piece_plane + king_bucket * PS_NB` where + `(sq ^ mask) + piece_plane <= 703 < PS_NB (704)`. So the P-factor + index is exactly `real_idx % 704` -- recoverable from the existing + `white_idx`/`black_idx` tensors at train time, in `nnue_model.py`. +2. **`nnue_model.py`**: add a second embedding table, + `self.ft_p = nn.EmbeddingBag(704, l1, mode="sum")`, **zero-initialized** + (matters: both tables sum into the same accumulator, so a + default-init P table would double the accumulator's scale at step + 0, confounding the factorization effect with an effective LR + change), with no bias of its own (`ft_bias` stays as the only + bias, or coalescing below isn't exact). In `accumulator()`, derive + `white_idx % 704` / `black_idx % 704` and sum `ft_p`'s output into + each perspective's accumulator alongside `ft`. +3. **Skip the "K" and "HalfRelativeKP" factors** Stockfish's doc also + describes. HalfKAv2_hm already encodes the king directly (kings + are included in the `piece_map()` loop via `piece_plane()`'s + `PS_KING` case), so the K factor would be redundant here. One + variable at a time, per the incremental-change rule used + throughout this project -- P alone first, measure, then decide. +4. **Coalescing after training, in `export_nnue_weights.py`**: this is + the step most likely to silently fail, because a bad coalesce + produces a net that is *worse* than not factoring at all (the real + HalfKAv2_hm rows learn less because the factor absorbed the shared + signal) and the training loss curve won't show it either way. + `ft.weight += ft_p.weight[torch.arange(22528) % 704]`, then drop + both `ft_p` and `material_probe` from the exported weights (two + training-only heads now, same pattern). **Required check**: assert + the pre-coalesce factored forward pass and the post-coalesce + plain-`ft`-only forward pass agree to ~1e-5 on a real batch before + trusting the export. +5. **Evaluation protocol**: Stockfish's doc is explicit that the + factorizer's effect is concentrated in *early* training and fades + as training progresses -- compare early-epoch loss descent against + a matched-seed, matched-epoch unfactored control, not just final + validation loss. A final-loss-only comparison could read a real + early-training effect as zero. + +### Sequencing + +This factorization work is queued behind the in-progress Stockfish +relabel -- the two are independent, complementary levers (ground-truth +quality vs. training-signal density under sparsity), and per this +project's incremental-change discipline they should be evaluated one +at a time rather than conflated in a single before/after comparison. + +## Status (2026-10-18, continued): Stockfish-labeled corpus built, P-factor +## factorization implemented and measured -- why it's load-bearing, not optional + +Both levers above are now done, in sequence, against the new +`pgn/stockfish_self_play_training_data.pgn` corpus (35,295 Stockfish- +vs-Stockfish games, 4,008,045 positions, labeled via Stockfish's own +search rather than this engine's `Eval()` -- see `extract_selfplay_ +positions.py`'s `MATE_CLAMP_THRESHOLD=90.0`, needed because this +foreign-engine corpus encodes mate as a clamped float, `[99.00,99.99]` +pawns via PolyGlot's UCI-to-xboard bridge, not typhoon's own `+M`/`-M` +sentinel token). Full pipeline run at `lambda=0.75` (`target = +0.75*p_search + 0.25*r_outcome`, chosen over the Phase-A `1.0` default +specifically because pure single-engine-distillation was the leading +suspect for the prior two nets' failure to beat HCE): `split_holdout.py` +-> `fit_k_and_label.py` (K=2.4130, this corpus's own fit, NOT the TWIC- +fit `DEFAULT_K=1.3123` -- the two corpora have different score +distributions and must not share a K) -> `quiet_filter_games.py` -> +`build_training_tensors.py` (now with `--workers`, default 16, +multiprocessing-sharded across positions, verified byte-for-byte +identical output to the old sequential path -- the per-position +feature-index computation has no cross-position dependency, so this +was a clean win on an idle multi-core box). + +**Why factorization is necessary here, not a nice-to-have**: measured +directly on this corpus's real training tensors (not a TWIC estimate), +2,237 of HalfKAv2_hm's 22,528 rows (9.9%) never fire once across all +2.84M training positions -- confirmed, after training, to be **bit- +for-bit identical** to their random pre-training init (diffed the +trained checkpoint's `ft.weight` against a freshly re-seeded, +untrained `NnueNet()` at the same seed: max difference on those rows +is exactly 0.0, vs. a clearly nonzero shift on every observed row). +This 9.9% figure is *better* than TWIC's 23.9% (this corpus's +decisive, played-to-conclusion games naturally cover more king +positions than human-resignation-biased TWIC), but still real, and +**not fixable by collecting more data at any practical scale**: +subsampling the existing pool at increasing sizes and fitting the +shrinkage curve (`zero_count ~= 5529 - 226*ln(N)`, log-linear, fit on +6 points from 10% to 100% of the current 2.84M-position pool) says +halving the dead-cell count to ~1,118 needs ~290M positions (~100x +the current pool), and getting under 500 needs ~4.4B (~1,560x). Brute- +force self-play volume cannot close this gap; it needs an +architecture-side fix, which is exactly what factorization is for. + +**Implementation** (`nnue_model.py`, `export_nnue_weights.py`), per +the plan above, built essentially as specified: a second, zero- +initialized `ft_p` `EmbeddingBag(PS_NB=704, l1)`, summed into the +accumulator alongside `ft` via `idx % PS_NB` (same bag offsets, no +`build_training_tensors.py` changes needed), coalesced into `ft.weight` +at export time (`ft.weight += ft_p.weight[arange(22528) % 704]`) with +a required pre/post-coalesce forward-pass agreement check (measured +3.58e-07 on a random-weighted smoke test, comfortably inside the +~1e-5 tolerance) before `ft_p` is dropped from the exported file. + +**Measured effect, matched-seed/epoch (lambda=0.75, K=2.4130, +material-loss-weight=0.001 -- see the next note on why that changed +from 0.01, 34 epochs, otherwise identical hyperparameters)**: +- Holdout MSE at matched checkpoints: factored beats unfactored at + every logged step, not just early training (contra the "effect + fades" caveat in the plan above, which turned out not to dominate + here) -- e.g. step 4000/epoch 11: 0.01193 (factored) vs. 0.01446 + (unfactored); final epoch 33: 0.01201 (factored) vs. 0.01331 + (unfactored). +- `sanity_check_net.py --cross-sectional` (2000 sampled holdout + positions, final checkpoints): both nets monotonic in piece value, + but factored's queen/pawn ratio is closer to nominal (~8.1x own / + 9.8x opponent-removed, vs. the already-decent unfactored run's + ~8.2x/9.9x -- both well above the old TWIC-trained matprobe net's + ~5.6x). +- Compared against a REAL Stockfish net's NNUE output (see next + section) on 1000 shared holdout positions: factored wins on every + metric -- correlation 0.879 vs 0.861, mean|diff| 0.776 vs 0.852 + pawns raw, 0.718 vs 0.762 pawns after fitting away the overall + scale difference (both nets run at ~0.79-0.83x Stockfish's scale, + expected since ours is WDL-sigmoid-calibrated, not raw-centipawn- + trained). + +**`material-loss-weight` lowered from 0.01 to 0.001 for this corpus**, +before any of the above: the original 0.01 (see "material-blindness +fixed" status above) was tuned against TWIC's homogeneous, balanced- +middlegame material distribution, where the raw material-MSE term +was naturally small and its weighted contribution stayed well below +the WDL loss's. This corpus has far more material-imbalanced +positions by construction (decisive self-play games played to real +conclusions, not human-resignation-biased), so the same 0.01 weight +measurably put the material term at parity with the WDL loss rather +than a gentle nudge -- confirmed by briefly launching training at +0.01 before catching it (checkpoints `ckpt_epoch000-001.pt` from that +run were deleted, not used for anything above). + +**Known residual gap, not yet fixed**: `ft_p`'s zero-init was +deliberate (avoids confounding the factorization effect with an +init-time accumulator-scale change), but `ft` itself still uses +`nn.EmbeddingBag`'s default `N(0,1)`-per-entry init, which is NOT +zero -- so an untouched row's random contribution is the same +magnitude (`norm~16`) as a fully-trained row's, not suppressed by +scale. Today this is masked by `ft_p`'s well-trained, shared +correction dominating the sum at those positions (confirmed: 0/1000 +Stockfish-comparison positions activated any dead cell, so this gap +hasn't tainted any reported result above), but it's a real residual +risk for any future position that activates several dead cells +together. Cheap, not-yet-applied fix: zero-init `ft` the same way +`ft_p` already is, so a dead cell degrades gracefully to "contributes +nothing" rather than "contributes untrained noise." + +## Verifying against a real Stockfish net (one-time diagnostic, not a +## pipeline dependency) + +To sanity-check whether the above was in the right ballpark at all (as +distinct from reusing Stockfish's own weights, which was explicitly +ruled out -- the goal is a from-scratch net strong enough to self-play +and iterate on, "eating its own tail," not a transplant), pulled a +real, period-appropriate Stockfish net (`nn-fdc1d0fe6455.nnue`, added +to Stockfish 2023-06-12 per `github.com/official-stockfish/networks`' +commit history, rev `38e61663d836e062af0bc002814ad5149c4b7729`) and +parsed its `FeatureTransformer::psqtWeights` table directly, against +that exact commit's header-reading code (fetched and read, not +guessed -- a first attempt at parsing by eye got the header layout +wrong and was caught before trusting any conclusion from it). + +- **Confirmed identical feature space**: `HalfKAv2_hm::HashValue = + 0x7f234cb8` and `Dimensions=22528` match our own spec exactly (same + king buckets, same orientation table, same piece-square planes); + this net's stored `ft` hash decodes to `L1=1536` (vs. our 256) via + `ft_hash = HashValue XOR (2*L1)`. +- **`psqtWeights[index][bucket]` for our own 2,237 dead-cell indices + looks completely ordinary** relative to that plane's overall average + (e.g. queen plane: 2487cp mean on dead cells vs. 2541cp overall) -- + i.e. our dusty corners aren't inherently weird positions, just a + sampling gap in this specific corpus, which is exactly the pattern + factorization is built to exploit (the true per-piece value is + roughly king-bucket-independent). +- To get a real NNUE eval from this old net for the comparison above, + built Stockfish from source at the exact matching commit + (`gmake -j$(nproc) build ARCH=x86-64-avx2 COMP=clang` -- note + FreeBSD's default `make` is BSD make and cannot parse this + Makefile's GNU-Make syntax, must use `gmake`), rather than risk + hand-reimplementing its 8-layer-stack quantized forward pass. + Compared its `eval` command's "NNUE evaluation" line (pure NNUE + output, no classical-blend) against our own `forward()`, both + converted to white-relative, over 1000 shared holdout FENs -- see + `annotate_pgns/compare_vs_stockfish_net.py` and the factorization + status note above for the numeric results. + +## Iterating on a net: adding a new generation of self-play data + +The first net to actually beat (or seriously contest) HCE at matched +`sd 10` is the `lambda=0.75`, P-factored checkpoint from the status +above -- the point where self-play with this net starts producing a +genuinely bigger/more-targeted training pool becomes real ("eating its +own tail", per the conversation that drove this project's current +phase). This section is the checklist for folding a new batch of +self-play games into the next training generation -- written down so +it doesn't need re-deriving from scratch (or re-making the same +mistakes) each time. **None of this requires a new architecture or a +new `nnue_model.py` -- same model class, same feature set, same +`--resume` mechanism.** It's a data/calibration discipline problem, +not a code problem. + +### 1. Combine, don't just append + +Pool the new self-play positions together with the existing training +corpus (corpora can mix freely at the games.jsonl level -- see +`extract_selfplay_positions.py`'s docstring) and re-run the **full** +pipeline on the combined set: + +``` +split_holdout.py -> fit_k_and_label.py -> quiet_filter_games.py -> build_training_tensors.py +``` + +Do NOT train on only the new self-play batch by itself, even via +`--resume`. Self-play games cluster around positions the current net +already favors (that's what self-play means) -- training on only the +new batch narrows the effective training distribution and risks +forgetting what the original, more diverse corpus taught. The combined +pool is also exactly how the dusty-corner coverage problem (see the +factorization status above) actually improves over time: new self-play +games sample different king positions/endgame structures than the +corpus that came before, so the pooled coverage count should be +re-measured after each new generation (same `np.bincount` check used +throughout this doc) to see whether it's actually shrinking the dead- +cell count, not just assumed. + +### 2. Refit K on the combined pool -- never carry an old K forward + +`fit_k_and_label.py`'s K is fit to one specific corpus's own score +distribution. **"Fitted K = 2.4130 -- record this; applying a different +K later (e.g. after labeling more data) is not the same model"** (the +tool's own exit message) applies in the other direction too: once the +pool changes, K must be refit on the new combined TRAIN split, not +reused from the previous generation. Training against a target +calibrated for a different score distribution than the one actually +being trained on silently miscalibrates every label, the same class of +problem as DNA-tuning's holdout-leakage concerns elsewhere in this +project. + +### 3. Watch the label source -- this is the one that actually matters + +How were the new self-play games scored? + +- **If an independent, stronger reference (Stockfish, or any engine + not being trained) produced the centipawn labels**, this is safe + regardless of which engine(s) played the moves -- self-play between + two candidates is a great way to source diverse, targeted positions + (specifically ones that stress whatever the current net is weakest + at), as long as the position doesn't then get "approved" by asking + the same net being trained to grade itself. +- **If the candidate net's own search produced the score labels for + its own games**, this reintroduces the single-generation self- + distillation problem already diagnosed as the likely reason the + first two (pre-Stockfish-relabel) nets failed to beat HCE at all + (see the "Revised understanding of why" status note above) -- a net + trained to reproduce its own judgment can at best match that + judgment, never exceed it. Mitigate, don't ignore, by keeping + `lambda < 1.0` in `build_training_tensors.py` so real game outcomes + (`r_outcome`) stay in the training target regardless of where the + centipawn half of the blend came from -- outcomes are ground truth + no self-distillation loop can corrupt, even when the score labels + themselves are suspect. + +### 4. Re-check `material-loss-weight` against the new pool's material + distribution + +`material-loss-weight` was lowered from `0.01` (tuned for TWIC's +balanced-middlegame distribution) to `0.001` specifically because this +project's Stockfish self-play corpus has far more decisive, material- +imbalanced games. A new self-play generation's material distribution +is a new empirical fact, not something to assume stays `0.001` forever +-- if candidate-vs-candidate games turn out far more drawish/balanced +than the original bootstrap corpus (plausible once both sides are +playing well), the same weight could again land at an unintended +fraction of the total loss. Check, don't assume -- same reasoning as +the original 0.01->0.001 change, just potentially in either direction +next time. + +### 5. Warm-start, don't restart + +`--resume <previous checkpoint>` on the new combined-pool tensors is +the right default -- it's strictly faster than random-init-and-train- +from-scratch, and there's no evidence in this project so far that +warm-starting hurts final quality (nothing here has tested the +opposite, to be clear -- if a future generation's holdout metrics look +worse than a from-scratch control at the same epoch count, that's +worth isolating before trusting warm-starting blindly forever). + +### 6. Validate before trusting the new generation + +In order, cheapest/fastest first -- stop at the first one that fails +rather than running all of them blind: + +1. `sanity_check_net.py --cross-sectional` on the new checkpoint -- + monotonic piece-value deltas, reasonable magnitudes (compare + against the queen/pawn ratio of the previous generation as a + baseline, not just "is it monotonic"). +2. Re-run the dead-cell coverage check (bincount over the new combined + training tensors) -- confirm it's trending down, not flat or up. +3. `compare_vs_stockfish_net.py` against the same real Stockfish net + used above, same held-out FEN sample where practical, to see + whether correlation/scale-fit-residual improved or regressed vs. + the previous generation's numbers (0.879 corr / 0.718 pawns resid. + as of the factored lambda=0.75 checkpoint -- the number to beat). +4. `export_nnue_weights.py` (coalesce check must pass, zero clipped + values) then a real matched-depth (`sd 10`, not `st` -- see the + fixed-depth-for-ECM-comparisons convention used throughout this + project) head-to-head via `eval_tune/test_vs_head.sh` with + `NNUE_WEIGHTS` set, against classic HCE. This is the only step that + actually answers "did this generation get better," everything above + is a cheaper proxy that can catch a regression before burning a + multi-hour match run on it. + +## Adoption criterion: sd-equal parity is not the bar, `st`-equal is -- +## and the net is currently ~3.3x slower per node than HCE + +Measured directly in the first real sd10-matched match (the factored +`lambda=0.75` checkpoint vs. classic HCE, 2026-10-18, 400 games): +candidate nps=470-475K vs. head nps=1.5-1.6M -- **a ~3.3x gap**, not a +rounding error. At matched *depth* this is invisible (both sides reach +sd10 regardless of nps), which is exactly why the sd-equal match is +the right first filter -- it isolates "is the net smarter per node" +from the speed handicap. But a `st`-equal (real time control) match +would currently be lopsided in HCE's favor for a reason that has +nothing to do with eval quality: at this project's branching factor +(ebf ~3.8-3.9, measured in the same match), a 3.3x nps disadvantage +costs roughly a full ply of search depth at any fixed time budget, on +top of whatever the eval-quality gap already costs. Testing at `st` +before closing that gap would answer "how bad is an engine with no +lazy-eval fast path," not "is this net any good." + +**Revised understanding (2026-10-18, same conversation): nps parity is +sufficient, not a blocker -- the real Stockfish binary on this exact +box is the proof.** It searches at ~500K nps -- essentially the same +as our net's measured 470-480K, not the 1.5-1.6M HCE achieves -- and +still beats this project's HCE every single recorded game (not even a +draw). Stockfish doesn't out-node HCE; it out-*prunes* it, searching +dramatically trimmer trees at the same or worse raw nps, because its +eval is trustworthy enough to prune aggressively on. This reframes the +priority between the two levers below: **lazy eval (closing the 3.3x +nps gap) is a nice-to-have, not the thing standing between this net +and real strength.** The actual lever is section 3 of the checklist +below -- null-move R, LMR's reduction table and fail-high prediction, +LMP thresholds, futility margins -- all of which currently encode how +*little* HCE's eval can be trusted. A learned eval earning the right to +prune harder, the way Stockfish's does, is plausibly worth more than +recovering the nps gap ever could be. Lazy eval stays worth doing (free, +no downside), just not gated-on-first the way the staged list below +originally implied. + +**Staged gate, in order -- do not skip ahead:** + +1. **Sd-equal match (done, 2026-10-18)**: necessary-but-not-sufficient + filter. If the net can't hold its own at matched depth, it has no + chance at matched time, since real time controls only make the + speed handicap worse for it, never better. +2. **Recalibrate the pruning-aggressiveness constants (checklist + section 3) -- this is the primary lever, per the Stockfish + comparison above, not a secondary cleanup pass.** Not a full + DNA-tuning sweep, just enough that search isn't making prune/ + reduce/extend decisions calibrated for a different eval's scale and + trustworthiness. Plausibly worth more than nps recovery on its own. +3. **Close the free speed gap too** -- lazy eval (checklist section 1), + still worth doing since it's pure engineering with no downside (it + doesn't change what the net *thinks*, only how often it's asked to + think at nodes that get pruned anyway) -- just not the bottleneck + the Stockfish comparison suggested it might be. +4. **Only then, a real `st`-equal SPRT against HCE.** This is the step + that actually answers the adoption question -- everything before it + answers a necessary precondition, not the question itself. + +**The adoption threshold itself shouldn't be pure parity-or-better, +though.** HCE is a mature, hand-tuned eval already deep into +diminishing returns on its own DNA-tuning pipeline. NNUE has an +open-ended improvement path still mostly unexploited (more self-play +generations per the iteration section above, factorization already +landed, a wider L1 if a Phase B GPU run ever happens -- see the real +Stockfish net's L1=1536 vs. our 256 as a sense of the room available). +A net that's close-but-not-quite-parity at a genuinely fair (`st`- +equal, post-calibration) time control may still be worth adopting if +the plan is to keep training it -- the alternative is maintaining two +eval paths indefinitely while one's ceiling keeps rising and the +other's doesn't. Close-and-improving beats even-and-stagnant. + +## Engine calibration checklist: what needs re-tuning for a new eval + +This engine's search has several subsystems whose margins/tables were +fit against classic `Eval()`'s specific output scale and noise +profile. None of this is required for the net to *run* (it already +does, correctly, per the wiring in `search.c`/`nnueeval.c`) -- it's +required for the net to run *well*, since an uncalibrated search is +fighting the new eval with pruning/reduction decisions tuned for a +different one. Confirmed against this engine's actual current search +(2026-10-18 conversation; EFP and ProbCut were previously assumed +present and are not -- corrected below rather than left as dead +checklist items): + +**1. Free speed, no behavior change (do this first, before any of the +below -- see the adoption-criterion note above for why ordering +matters here):** +- Lazy eval fast path -- designed (architecture decisions above: reuse + the net's own cheap linear/PSQT-ish term -- here `ft_p`/ + `material_probe`-shaped -- as a first-pass filter, falling through + to the full forward pass only when inconclusive vs. alpha/beta), not + yet built. This is the single biggest lever on the measured 3.3x nps + gap and touches zero tuning -- it's the right first move specifically + because it can't make the net play worse, only faster. +- Deferred/lazy accumulator maintenance (flagged in the accumulator + design section above as a "performance follow-up, not required for + correctness") -- avoid eager accumulator updates at nodes pruned + before any eval is ever consulted. + +**2. The raw-score-to-cp mapping -- verify this before touching any +margin below, since it's a multiplier on all of them:** +Every margin in section 3 is a flat cp (or near-cp) threshold tuned +against HCE's output scale. The trained net runs at ~0.8x a real +Stockfish net's scale (measured via `compare_vs_stockfish_net.py`) and +is WDL-sigmoid-calibrated (K=2.4), not raw-centipawn-trained. If +search currently treats `forward()`'s raw output as if it were +already in HCE's cp convention, every margin below is off by that +scale factor before even considering noise-profile differences. This +needs the inverse-sigmoid-to-cp conversion the original architecture +decision already called for ("Requires a calibrated inverse-sigmoid +conversion back to centipawns... that conversion constant needs its +own calibration pass") -- confirm it exists and is right once, rather +than chasing a scale error through a dozen separately-miscalibrated +margins. + +**3. Score-scale/noise-dependent pruning and reduction constants -- +this engine's actual current set (corrected 2026-10-18: no EFP in main +search, only LMP; no separate razoring step -- LMR fills that role +here; no ProbCut; QSearch futility and "delta pruning" are the same +mechanism, not two):** +- QSearch futility margins +- Static/reverse null-move pruning margin +- Null-move reduction depth R, and any eval-based skip/verification + gating on the null-move try itself +- LMP move-count table, `data.c` +- LMR reduction table, `data.c`, **and** the joint fail-high + prediction table feeding it (see the LMR sd11/sn10M protocol note + elsewhere in this project) -- this table is a direct statement about + how trustworthy the eval is at a reduced-depth node, fit entirely + against HCE's noise profile +- Root aspiration window initial width and re-widening schedule on + fail-high/fail-low (tuned against HCE's iteration-to-iteration score + noise -- a good catch, easy to overlook since it's at the root, not + buried in the main search loop) + +**4. Eval-component-dependent heuristics (lower priority -- already +confirmed to degrade gracefully under a lazy/missing full eval per the +"Breaking external dependencies" section above; re-measure how often +they actually fire once NNUE lazy eval ships, don't re-derive from +scratch):** +- `EstimateHangingMaterialCp`'s `cTrapped`-fallback "dumb move" axis +- `ValueOfMaterialInTroubleDespiteMove`'s QSearch stand-pat gate + +**5. Currently dormant, no action needed yet:** draw/contempt scoring +is built but always returns 0 right now -- no live miscalibration risk +today, but if it's ever turned on, the same cp-scale concern in +section 2 applies to whatever constant drives it. -## Immediately actionable, unblocked work +## Status (2026-10-18, continued): pipeline-scaling audit, three gaps fixed, end-to-end driver added -Not gated behind any engine-internals decision above: +Prompted by generating a much larger volume of annotated PGN than +before and wanting to confirm the post-annotation Python pipeline +(`annotate_pgns/`) actually uses all available parallelism rather than +silently bottlenecking on a single-threaded stage. `annotate.py` +itself (the sd-N search stage) was already confirmed fine -- sharded +across N single-threaded engine subprocesses -- and +`build_training_tensors.py` already had a `--workers 16` +`multiprocessing.Pool`. Three real gaps found and fixed; one +unrelated question (an end-to-end PGN-to-`.nnue` driver) answered by +adding a new orchestrator script. -- Build the annotation harness and begin scoring `lichess_elite_*`/ - `twic_filtered.pgn` games (Phase 1 training data). -- Investigate `<TT>`/`<EGTB>`/`<REC>` tag semantics in the self-play PV - printer. -- Finalize the written HalfKAv2_hm feature-indexing spec (design-only, - no code). +- **Gap: `quiet_filter.py` had no parallelism at all.** It's a + single-threaded Python loop doing `chess.Board()` construction + SEE + per row over the entire corpus (2.5M+ rows for the Stage A run) -- + the one CPU-bound step in the non-annotation part of the pipeline + with zero parallelism, unlike `build_training_tensors.py`'s `Pool` + right next to it. **Fixed via a new wrapper, `shard_quiet_filter.py`**, + rather than rewriting `quiet_filter.py` itself: round-robin-splits + the input across `--workers` (default 16) temp shard files, runs + that many `quiet_filter.py` subprocesses concurrently, concatenates + their outputs. Deliberately reuses `quiet_filter.py` unmodified -- + one definition of the filter, reused as a subprocess, the wrapper + only owns sharding/recombining. Row order in the combined output + does not match the input (shards are concatenated shard-by-shard, + not interleaved back to original order) -- harmless here since + `positions.quiet.jsonl` is a flat, order-independent training-set + input. Verified end-to-end on a synthetic 40-row fixture: 40 in, 40 + out, across 4 shards. +- **Gap: `fit_k_and_label.py`'s ternary-search K-fit was unvectorized + Python.** `fit_k()`'s 40 ternary-search iterations each call `mse()` + twice, and the old `mse()` was a plain Python `for` loop over every + (score, outcome) pair -- roughly 80 full passes over ~2M+ pairs in + pure Python for a step that does very little actual compute. + **Rewritten to numpy** (`sigmoid`/`mse` as vectorized array ops over + the full pair set) **and sharded** across `--workers` (default 16) + via `multiprocessing.Pool`, for *both* halves of the file: the + calibration-pair collection (`collect_calibration_pairs`, now + `collect_chunk` workers each vectorizing one chunk of games with + numpy, combined via `np.concatenate`) and the labeling pass + (`label_file`, now `label_chunk` workers, each vectorizing one + game's non-exact positions' sigmoid computation per chunk) -- + `Pool.imap`-based, order-preserving, mirroring + `build_training_tensors.py`'s existing chunking pattern. Verified + end-to-end on a synthetic 50-game fixture (4 workers, chunk-size 7): + correct game/position counts, K fit, both splits labeled. +- **Gap: `nnue_train.py`'s CPU thread default was lower than the rest + of the pipeline's convention.** `--threads` default bumped 8 -> 16, + matching the `--workers 16` convention used everywhere else in this + pipeline (kept below this box's 24 cores -- see below -- rather than + maxed out, so other jobs on the shared box keep some breathing + room). +- **Core-count note, not acted on:** this box has 24 cores + (`nproc`), so every one of these `--workers`/`--threads`/ + `annotate.py --shards` defaults (16) intentionally leaves some + headroom rather than using all 24 -- a deliberate choice to leave + the box's other jobs breathing room, not an oversight. `annotate.py` + Stage A's own measured throughput (NNUE.md's "Status (2026-10-14)" + section: ~75 pos/sec aggregate at 16 shards vs. ~112/sec + extrapolated from one shard x16, attributed to contention) is a + reminder that this box's parallelism doesn't scale linearly past a + point anyway -- blindly raising shard/worker counts further is not + assumed to help without measuring first. +- **New: `run_nnue_pipeline.py`**, an end-to-end orchestrator from a + glob of input PGN to a `.nnue` weights file, requested directly as + "can one script run the whole training process end to end." Targets + **already-scored PGN** specifically (match_play.py's + `"{+-}NN.NN/DEPTH PV..."` per-move comment format -- e.g. + `pgn/stockfish_self_play_training_data.pgn`, `pgn/self_play_games.pgn`), + not raw human-game PGN needing a fresh sd-N search -- that's + `annotate.py`'s job and is a different, not-yet-automated entry + point. Chains, in order: `extract_selfplay_positions.py` (once per + input file, since that script takes a single PGN; outputs + concatenated) -> `split_holdout.py` -> `fit_k_and_label.py` -> + `quiet_filter_games.py` (both splits) -> `build_training_tensors.py` + (both splits) -> `nnue_train.py` -> `export_nnue_weights.py` (against + the latest `ckpt_epochNNN.pt` written -- `nnue_train.py` has no + separate "best checkpoint" selection yet, so "latest" is the only + option today). Every stage checks whether its expected output + already exists and skips it unless `--force` is passed -- the later + stages here can run for hours, so a crash/interrupt partway through + should not mean restarting from stage 1. Passes `--workers`/ + `--threads` through at the same default of 16 used everywhere else. + A thin orchestrator by design: shells out to the existing, + individually-tested scripts via `subprocess` rather than + reimplementing any of their logic. Argument parsing, `--help`, and + the glob-miss error path were smoke-tested; the full multi-stage run + (which needs the engine binary, real PGNs, and hours of wall clock) + was not run end-to-end as part of this change. -Gated behind opening a dedicated branch (per discussion -- not yet -started): +## TODO (not yet implemented) -- Any `SEARCHER_THREAD_CONTEXT`/`PLY_INFO`/MakeMove/UnmakeMove changes. +- **Horizontal-flip data augmentation, in the position loader + (`build_training_tensors.py`).** HalfKAv2_hm's king-bucket mirroring + does NOT tie piece-square weights across the a/h files -- the + `OrientTBL` XOR step re-orients the board onto the king's own half + but a piece's *file within that half* still maps to a distinct + feature row, so a↔h-mirrored positions are not already folded + together by the architecture. Randomly mirroring positions + left-right at load time (flip every square's file, recompute both + perspectives' indices, swap castling rights symmetrically where + applicable) is a legitimate, standard way to roughly double the + effective sample count per feature row without changing dataset + size on disk. Directly relevant given the measured sparsity + (23.9% of HalfKAv2 rows with zero observations, per the + 2026-10-18 factorization status above) -- should be evaluated + alongside P-factorization, not instead of it, since the two + address the same sparsity symptom through different mechanisms + (more effective samples per row vs. a denser auxiliary feature + set) and their effects may not be additive. +- **L2 weight decay in `nnue_train.py`'s optimizer (AdamW).** Pulls + rarely-updated weights (the same sparse/underexposed rows the + factorization work above targets) toward zero instead of leaving + them to drift on whatever noisy gradient they happened to last + receive. Cheap -- a single optimizer hyperparameter, no + architecture or export-format change -- so there's no reason to + gate it behind the factorization/augmentation experiments above; + can be added and swept independently. @@ -3275,6 +3275,12 @@ Return value: static UINT64 g_u64SuperLazyEvalsByBucket[BASE_MARGIN_MAT_BUCKETS]; static UINT64 g_u64LazyEvalsByBucket[BASE_MARGIN_MAT_BUCKETS]; static UINT64 g_u64FullEvalsByBucket[BASE_MARGIN_MAT_BUCKETS]; +// Same idea, for the two NNUE exits -- added 2026-10-16 alongside the +// EVAL_POSITIONAL_SOURCE_NNUE_FULL/NNUE_LAZY tiers, specifically to +// answer "what fraction of real NNUE eval calls does the lazy probe +// gate actually resolve" with a real measurement instead of a guess. +static UINT64 g_u64NnueLazyEvalsByBucket[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_u64NnueFullEvalsByBucket[BASE_MARGIN_MAT_BUCKETS]; void DumpEvalExitRateByBucket(void) @@ -3324,6 +3330,29 @@ Return value: 100.0 * (double)u64L / (double)u64Total, 100.0 * (double)u64F / (double)u64Total); } + + Trace("NNUE eval exit rate by material bucket (lazy probe gate vs full " + "forward pass):\n"); + for (m = 0; m < BASE_MARGIN_MAT_BUCKETS; m++) + { + UINT64 u64NL = g_u64NnueLazyEvalsByBucket[m]; + UINT64 u64NF = g_u64NnueFullEvalsByBucket[m]; + UINT64 u64Total = u64NL + u64NF; + if (0 == u64Total) + { + Trace(" bucket %u (scaler %u-%u): no samples\n", + m, m * BASE_MARGIN_MAT_WIDTH, + (m * BASE_MARGIN_MAT_WIDTH) + BASE_MARGIN_MAT_WIDTH - 1); + continue; + } + Trace(" bucket %u (scaler %2u-%2u): n=%12" COMPILER_LONGLONG_UNSIGNED_FORMAT + " nnue_lazy=%5.2f%% nnue_full=%5.2f%%\n", + m, m * BASE_MARGIN_MAT_WIDTH, + (m * BASE_MARGIN_MAT_WIDTH) + BASE_MARGIN_MAT_WIDTH - 1, + u64Total, + 100.0 * (double)u64NL / (double)u64Total, + 100.0 * (double)u64NF / (double)u64Total); + } } @@ -3494,6 +3523,37 @@ static UINT64 g_uSuperLazyMarginSwingSumSq[BASE_MARGIN_MAT_BUCKETS]; static UINT64 g_uSuperLazyMarginExceeded[BASE_MARGIN_MAT_BUCKETS]; static UINT64 g_uSuperLazyMarginExceededFenSamples[BASE_MARGIN_MAT_BUCKETS]; +// Same again, for the experimental NNUE lazy probe (docs/NNUE.md's +// "Lazy-evaluation tiering plan") -- |NnueForward() - NnueProbeScore()| +// swing, same bucketing (_MaterialBucket), same harness, own counter +// set so this doesn't mix with either HCE tier's numbers. No real +// margin exists for this tier yet -- this is purely measurement, not +// gating anything -- so RecordNnueLazyMarginSafetySwing is always +// called with an effectively-infinite iActualMarginUsed (see its call +// site in Eval()) until a real margin is derived from this data. +static UINT64 g_uNnueLazyMarginSwingMax[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueLazyMarginSwingCount[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueLazyMarginSwingSum[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueLazyMarginSwingHist[BASE_MARGIN_MAT_BUCKETS][SWING_HIST_BINS]; +static UINT64 g_uNnueLazyMarginSwingSumSq[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueLazyMarginExceeded[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueLazyMarginExceededFenSamples[BASE_MARGIN_MAT_BUCKETS]; + +// Material-only "super lazy" tier -- |NnueForward() - +// pos->iMaterialBalance[uToMove]|, no fit/regression at all (unlike +// the accumulator probe above): VALUE_PAWN=100 means iMaterialBalance +// is already in centipawns, the same units NnueForward returns, so +// raw material balance is directly usable as the "center" estimate -- +// exactly how classic Eval()'s own super-lazy exit already uses it. +// Same bucketing/harness as the lazy tier above. +static UINT64 g_uNnueSuperLazyMarginSwingMax[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueSuperLazyMarginSwingCount[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueSuperLazyMarginSwingSum[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueSuperLazyMarginSwingHist[BASE_MARGIN_MAT_BUCKETS][SWING_HIST_BINS]; +static UINT64 g_uNnueSuperLazyMarginSwingSumSq[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueSuperLazyMarginExceeded[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uNnueSuperLazyMarginExceededFenSamples[BASE_MARGIN_MAT_BUCKETS]; + static void _RecordMarginSafetySwingInto(IN OUT UINT64 *puMax, IN OUT UINT64 *puCount, @@ -3616,6 +3676,84 @@ Return value: } +static void +RecordNnueLazyMarginSafetySwing(IN POSITION *pos, + IN SCORE iSwing, + IN SCORE iActualMarginUsed) +/** + +Routine description: + + Same shape as RecordSuperLazyMarginSafetySwing, but for the + experimental NNUE accumulator-based lazy probe (docs/NNUE.md's + "Lazy-evaluation tiering plan") -- |NnueForward() - + NnueProbeScore()|, own counter set. No margin is actually gated on + yet -- iActualMarginUsed is passed as an effectively-infinite + sentinel from Eval() until a real margin is derived from this + measurement, so puExceeded stays meaningless for now; only the + histogram/sum/sumSq/max/count fields are real data at this stage. + +Parameters: + + POSITION *pos + SCORE iSwing : >= 0 + SCORE iActualMarginUsed : sentinel (no real margin chosen yet) + +Return value: + + void + +**/ +{ + _RecordMarginSafetySwingInto(g_uNnueLazyMarginSwingMax, + g_uNnueLazyMarginSwingCount, + g_uNnueLazyMarginSwingSum, + g_uNnueLazyMarginSwingSumSq, + g_uNnueLazyMarginSwingHist, + g_uNnueLazyMarginExceeded, + g_uNnueLazyMarginExceededFenSamples, + "nnue_lazy", + pos, iSwing, iActualMarginUsed); +} + + +static void +RecordNnueSuperLazyMarginSafetySwing(IN POSITION *pos, + IN SCORE iSwing, + IN SCORE iActualMarginUsed) +/** + +Routine description: + + Same shape as RecordNnueLazyMarginSafetySwing, but for the + material-only "super lazy" signal (|NnueForward() - + pos->iMaterialBalance[uToMove]|) -- own counter set, same + sentinel-until-a-real-margin-exists convention. + +Parameters: + + POSITION *pos + SCORE iSwing : >= 0 + SCORE iActualMarginUsed : sentinel (no real margin chosen yet) + +Return value: + + void + +**/ +{ + _RecordMarginSafetySwingInto(g_uNnueSuperLazyMarginSwingMax, + g_uNnueSuperLazyMarginSwingCount, + g_uNnueSuperLazyMarginSwingSum, + g_uNnueSuperLazyMarginSwingSumSq, + g_uNnueSuperLazyMarginSwingHist, + g_uNnueSuperLazyMarginExceeded, + g_uNnueSuperLazyMarginExceededFenSamples, + "nnue_superlazy", + pos, iSwing, iActualMarginUsed); +} + + // Plain Newton's-method sqrt, no <math.h> -- chess.h's own VALUE_KING // #defines INFINITY_SCORE as a board-eval sentinel, which collides with // <math.h>'s own INFINITY_SCORE macro, so libm's sqrt() isn't available @@ -3729,6 +3867,20 @@ Return value: g_uSuperLazyMarginSwingSumSq, g_uSuperLazyMarginSwingMax, g_uSuperLazyMarginSwingHist); + _PersistMarginSwingHistogramToFile(f, "nnue_lazy", + g_uNnueLazyMarginSwingCount, + g_uNnueLazyMarginExceeded, + g_uNnueLazyMarginSwingSum, + g_uNnueLazyMarginSwingSumSq, + g_uNnueLazyMarginSwingMax, + g_uNnueLazyMarginSwingHist); + _PersistMarginSwingHistogramToFile(f, "nnue_superlazy", + g_uNnueSuperLazyMarginSwingCount, + g_uNnueSuperLazyMarginExceeded, + g_uNnueSuperLazyMarginSwingSum, + g_uNnueSuperLazyMarginSwingSumSq, + g_uNnueSuperLazyMarginSwingMax, + g_uNnueSuperLazyMarginSwingHist); fclose(f); Trace("Raw margin-safety histograms written to %s\n", szPath); } @@ -3828,6 +3980,18 @@ Return value: g_uSuperLazyMarginSwingSum, g_uSuperLazyMarginSwingSumSq, g_uSuperLazyMarginExceeded); + _DumpMarginSafetyCalibrationInto("NNUE lazy probe margin safety", + g_uNnueLazyMarginSwingMax, + g_uNnueLazyMarginSwingCount, + g_uNnueLazyMarginSwingSum, + g_uNnueLazyMarginSwingSumSq, + g_uNnueLazyMarginExceeded); + _DumpMarginSafetyCalibrationInto("NNUE super lazy (material-only) margin safety", + g_uNnueSuperLazyMarginSwingMax, + g_uNnueSuperLazyMarginSwingCount, + g_uNnueSuperLazyMarginSwingSum, + g_uNnueSuperLazyMarginSwingSumSq, + g_uNnueSuperLazyMarginExceeded); PersistMarginSafetyHistogramsToFile(szHistogramPathOverride); } #endif // CALIBRATE_MARGIN_SAFETY @@ -4104,7 +4268,7 @@ static const SCORE iKingP98ByAttackerArmy[8] = // p98/p2 pessimistic/optimistic pairing margin_a/margin_b use -- // p50 is the right statistic for a bias correction, p98/p2 is the // right statistic for a soundness bound. iPositional[color] = -// iKingP50ByAttackerArmyOnly[opponent's army bucket] (opponent is the +// iKingP50ByAttackerArmy[opponent's army bucket] (opponent is the // attacker of this side's own king) + iResidualP50ByOwnArmy[this // side's own army bucket] (or the super-lazy-scoped sibling, for the // super-lazy exit). @@ -7043,22 +7207,18 @@ Return value: // intentionally the least-specific answer that's still safe, not // an attempt to approximate what a real positional estimate would // say. - if ((TRUE == g_fUseNnueEval) && (TRUE == g_fNnueWeightsLoaded)) - { - SCORE iNnueScore = NnueForward(ctx); - if (NULL != piPositional) - { - (*piPositional)[WHITE] = 0; - (*piPositional)[BLACK] = 0; - } - return iNnueScore; - } - - // Initialize army scalers here (moved ahead of the super-lazy exit - // point below, 2026-09-08 -- their only inputs, uNonPawnMaterial, - // are already available this early, and the super-lazy margin - // table needs a material bucket before it can run). We skip lazy - // eval if we're too late into an endgame. + // Initialize army scalers here -- moved ahead of BOTH the NNUE + // short-circuit above and the super-lazy exit point below, + // 2026-10-15 (was already ahead of super-lazy only, since + // 2026-09-08; moved further to also cover the NNUE path). Their + // only inputs, uNonPawnMaterial, are already available this early. + // Side effect of this move: pos->uArmyScaler is now always fresh + // after Eval() returns, including on the NNUE path -- previously + // it was left stale (whatever a prior call last set) whenever + // Eval() took the NNUE short-circuit, since that return happened + // before this block ran at all. Nothing on the NNUE path + // previously depended on uArmyScaler being stale vs fresh, so this + // is strictly safer, not a behavior change worth gating. pos->uArmyScaler[BLACK] = pos->uNonPawnMaterial[BLACK] - VALUE_KING; pos->uArmyScaler[WHITE] = pos->uNonPawnMaterial[WHITE] - VALUE_KING; pos->uArmyScaler[BLACK] /= VALUE_PAWN; @@ -7076,6 +7236,143 @@ Return value: (SystemReadTimeStampCounter() - u64MaterialSetupTimer); #endif + if ((TRUE == g_fUseNnueEval) && (TRUE == g_fNnueWeightsLoaded)) + { + // p98 of |NnueForward() - NnueProbeScore()| per material + // bucket. Superseded 2026-10-15: originally derived from a + // 91M-sample population over three curated tactical suites + // (sd 10) -- replaced same day by a far larger and more + // representative 3.09-BILLION-sample population, 16 parallel + // shards of 12,500 positions each (200,000 total) from + // tests/cross_sectional_200k.ep_ at sd 6. Per-suite tactical + // positions are a narrow, biased slice (selected for sharp + // tactics); the cross-sectional pool's much broader position + // variety surfaced real outlier swings the narrower sample + // never saw (max up to 534cp here vs ~472cp before) while + // landing close to the same per-bucket numbers overall (mostly + // 0-10cp different) -- see docs/NNUE.md's "Lazy-evaluation + // tiering plan". Same bucketing as _MaterialBucket (combined + // army scaler, width BASE_MARGIN_MAT_WIDTH). + static const SCORE iNnueLazyMarginByMatBucket[BASE_MARGIN_MAT_BUCKETS] = + { + 180, 200, 190, 180, 170, 160, 150, 150, + }; + +#ifdef CALIBRATE_MARGIN_SAFETY + // Calibration build: ALWAYS run the real forward pass, never + // skip it -- so any candidate gate decision can keep being + // measured against ground truth before it's trusted live. + // When the gate is enabled, measure against the real chosen + // margin (making "exceeded" a meaningful, ongoing validation + // signal); when disabled, still measure against an + // effectively-infinite sentinel (pure observation, matches + // this harness's original pre-gate behavior). + SCORE iNnueScore = NnueForward(ctx); + if (TRUE == g_fNnueProbeLoaded) + { + SCORE iProbeScore = NnueProbeScore(ctx); + SCORE iSwing = (SCORE)abs((int)iNnueScore - (int)iProbeScore); + SCORE iMarginUsed = (TRUE == g_fUseNnueLazyProbe) ? + iNnueLazyMarginByMatBucket[_MaterialBucket(pos)] : + INFINITY_SCORE; + RecordNnueLazyMarginSafetySwing(pos, iSwing, iMarginUsed); + } + { + // Material-only "super lazy" measurement -- no fit, no + // probe, just the already-maintained material balance + // (same cp units as NnueForward's output, see + // RecordNnueSuperLazyMarginSafetySwing). No real margin + // chosen yet, so always the infinite sentinel for now. + SCORE iSuperLazyScore = pos->iMaterialBalance[pos->uToMove]; + SCORE iSuperLazySwing = + (SCORE)abs((int)iNnueScore - (int)iSuperLazyScore); + RecordNnueSuperLazyMarginSafetySwing(pos, iSuperLazySwing, INFINITY_SCORE); + } + // This branch always returns the real forward pass (never + // skips, by design -- see the big comment above), so it's + // unconditionally the NNUE_FULL tier for futility-margin + // selection purposes. + ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_FULL; + g_u64NnueFullEvalsByBucket[_MaterialBucket(pos)]++; + if (NULL != piPositional) + { + (*piPositional)[WHITE] = 0; + (*piPositional)[BLACK] = 0; + } + return iNnueScore; +#else + // Production build: if the lazy probe gate is enabled and + // confidently resolves the fail-high/fail-low question + // relative to [iAlpha, iBeta], skip the real forward pass + // entirely -- this is the actual speed win. Otherwise (gate + // disabled, probe not loaded, or inconclusive), fall through + // to the full pass exactly as before this feature existed. + // Super-lazy pre-filter, experimental -- 2026-10-16: cheapest + // possible check, pure material balance, no net evaluation at + // all. Previously measurement-only (CALIBRATE_MARGIN_SAFETY + // above records the real swing but always used an infinite + // sentinel margin, i.e. never actually gated). Deliberately + // loose/uncalibrated here per the "safety doesn't win games" + // experiment -- g_iNnueSuperLazyMargin is a flat runtime + // override (see nnueeval.c), not a per-bucket fit. + if (TRUE == g_fUseNnueSuperLazy) + { + SCORE iSuperLazyScore = pos->iMaterialBalance[pos->uToMove]; + + if (((iSuperLazyScore - g_iNnueSuperLazyMargin) > iBeta) || + ((iSuperLazyScore + g_iNnueSuperLazyMargin) < iAlpha)) + { + // Feed the same aggregate counters root.c's "Eval exit + // breakdown" line reads -- otherwise that line stays + // 0/0/0 under NNUE eval since it was only ever wired + // to the HCE's own super-lazy/lazy/full exit sites. + INC(ctx->sCounters.tree.u64SuperLazyEvals); + ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_LAZY; + g_u64NnueLazyEvalsByBucket[_MaterialBucket(pos)]++; + if (NULL != piPositional) + { + (*piPositional)[WHITE] = 0; + (*piPositional)[BLACK] = 0; + } + return iSuperLazyScore; + } + } + if ((TRUE == g_fUseNnueLazyProbe) && (TRUE == g_fNnueProbeLoaded)) + { + SCORE iProbeScore = NnueProbeScore(ctx); + SCORE iMargin = (g_iNnueLazyMarginOverride >= 0) ? + (SCORE)g_iNnueLazyMarginOverride : + iNnueLazyMarginByMatBucket[_MaterialBucket(pos)]; + + if (((iProbeScore - iMargin) > iBeta) || + ((iProbeScore + iMargin) < iAlpha)) + { + INC(ctx->sCounters.tree.u64LazyEvals); + ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_LAZY; + g_u64NnueLazyEvalsByBucket[_MaterialBucket(pos)]++; + if (NULL != piPositional) + { + (*piPositional)[WHITE] = 0; + (*piPositional)[BLACK] = 0; + } + return iProbeScore; + } + } + { + SCORE iNnueScore = NnueForward(ctx); + INC(ctx->sCounters.tree.u64FullEvals); + ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_FULL; + g_u64NnueFullEvalsByBucket[_MaterialBucket(pos)]++; + if (NULL != piPositional) + { + (*piPositional)[WHITE] = 0; + (*piPositional)[BLACK] = 0; + } + return iNnueScore; + } +#endif + } + #ifdef DUMP_RESIDUAL_SAMPLES // Super-lazy's OWN baseline -- material only, before _EvalPawns // even runs. Separate from iResidualBaselineWhite/Black below @@ -7178,7 +7475,7 @@ Return value: // 2026-09-17: no longer flat 0 -- same p50-based typical- // value estimate as regular lazy, using the super-lazy- // scoped residual table (material-only baseline). See - // iKingP50ByAttackerArmyOnly/iSuperLazyResidualP50By- + // iKingP50ByAttackerArmy/iSuperLazyResidualP50By- // OwnArmy (file-scope, above). ULONG uWhiteBucket = MINU(7, pos->uArmyScaler[WHITE] / 4); ULONG uBlackBucket = MINU(7, pos->uArmyScaler[BLACK] / 4); @@ -7520,7 +7817,7 @@ Return value: // terms (king safety + residual), matching full-eval's // own iPositional convention (that side's own real // positional score, not a relative-to-opponent - // quantity). See iKingP50ByAttackerArmyOnly/ + // quantity). See iKingP50ByAttackerArmy/ // iResidualP50ByOwnArmy (file-scope, above) for the // full percentile charts this was derived from. ULONG uWhiteBucket = MINU(7, pos->uArmyScaler[WHITE] / 4); @@ -7546,10 +7843,10 @@ Return value: iSavedLazyScore = iScoreForSideToMove; iSavedLazyPositional = iAlphaMargin; iSavedLazyPositionalWhite = - iKingP50ByAttackerArmyOnly[MINU(7, pos->uArmyScaler[BLACK] / 4)] + + iKingP50ByAttackerArmy[MINU(7, pos->uArmyScaler[BLACK] / 4)] + iResidualP50ByOwnArmy[MINU(7, pos->uArmyScaler[WHITE] / 4)]; iSavedLazyPositionalBlack = - iKingP50ByAttackerArmyOnly[MINU(7, pos->uArmyScaler[WHITE] / 4)] + + iKingP50ByAttackerArmy[MINU(7, pos->uArmyScaler[WHITE] / 4)] + iResidualP50ByOwnArmy[MINU(7, pos->uArmyScaler[BLACK] / 4)]; #else #ifdef EVAL_TIME @@ -7591,10 +7888,10 @@ Return value: iSavedLazyScore = iScoreForSideToMove; iSavedLazyPositional = iBetaMargin; iSavedLazyPositionalWhite = - iKingP50ByAttackerArmyOnly[MINU(7, pos->uArmyScaler[BLACK] / 4)] + + iKingP50ByAttackerArmy[MINU(7, pos->uArmyScaler[BLACK] / 4)] + iResidualP50ByOwnArmy[MINU(7, pos->uArmyScaler[WHITE] / 4)]; iSavedLazyPositionalBlack = - iKingP50ByAttackerArmyOnly[MINU(7, pos->uArmyScaler[WHITE] / 4)] + + iKingP50ByAttackerArmy[MINU(7, pos->uArmyScaler[WHITE] / 4)] + iResidualP50ByOwnArmy[MINU(7, pos->uArmyScaler[BLACK] / 4)]; #else #ifdef EVAL_TIME @@ -8127,10 +8424,10 @@ Return value: ULONG uWhiteBucket = MINU(7, pos->uArmyScaler[WHITE] / 4); ULONG uBlackBucket = MINU(7, pos->uArmyScaler[BLACK] / 4); (*piPositional)[WHITE] = - iKingP50ByAttackerArmyOnly[uBlackBucket] + + iKingP50ByAttackerArmy[uBlackBucket] + iSuperLazyResidualP50ByOwnArmy[uWhiteBucket]; (*piPositional)[BLACK] = - iKingP50ByAttackerArmyOnly[uWhiteBucket] + + iKingP50ByAttackerArmy[uWhiteBucket] + iSuperLazyResidualP50ByOwnArmy[uBlackBucket]; // CALIBRATE_MARGIN_SAFETY builds: the FULL tag set above // reflects the fallthrough that just ran for measurement diff --git a/src/eval_tune/build_cross_sectional_all.py b/src/eval_tune/build_cross_sectional_all.py new file mode 100644 index 0000000..144c87c --- /dev/null +++ b/src/eval_tune/build_cross_sectional_all.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Build a larger cross-sectional position pool (successor to +tests/cross_sectional_200k.ep_) out of the full twic.pgn + +lichess_elite*.pgn corpus. + +Unlike sample_eval_calibration_positions.py's random-byte-offset +sampling (one position per game, uniform-random ply), this does an +exhaustive streaming pass over every game and pulls 5 positions per +qualifying game at fixed fractional points in the game +(early/early-mid/mid/mid-late/late) -- Scott's own framing. Fractional- +ply sampling structurally balances game-stage representation (every +kept game contributes exactly one position to each of the 5 stage +buckets), which random-ply sampling does not. + +Deliberately does NOT filter for quiescence -- these positions are +`go` roots for the engine's own search (which resolves tactics via its +own qsearch at every node), not Texel-tuning eval-only positions. See +sample_eval_calibration_positions.py's own docstring for the same +reasoning: a cross-sectional pool should look like "all positions a +real search tree touches", not just the quiet subset. + +Ply fractions: 5 samples come from 6 equal segments (1/6..5/6 of the +game's ply count), not literal 1/5..5/5 -- 5/5 is the final move of +the game, frequently checkmate/stalemate/resignation-adjacent, which +`go` can't search meaningfully. Interior boundaries only. +""" +import argparse +import collections +import io +import sys +from pathlib import Path + +import chess +import chess.pgn + +STAGE_NAMES = ["early", "early_mid", "mid", "mid_late", "late"] +FRACTIONS = [1 / 6, 2 / 6, 3 / 6, 4 / 6, 5 / 6] + +PIECE_VALUES = {chess.KNIGHT: 3, chess.BISHOP: 3, chess.ROOK: 5, chess.QUEEN: 9} + + +def material_bucket(board): + """Same bucketing as eval.c's _MaterialBucket / the calibration + script's own copy: combined non-pawn army value (pawn units) / 8, + capped at 7 (0 = bare kings, 7 = full army).""" + total = 0 + for pt, v in PIECE_VALUES.items(): + total += v * len(board.pieces(pt, chess.WHITE)) + total += v * len(board.pieces(pt, chess.BLACK)) + return min(7, total // 8) + + +def iter_game_texts(pgn_path): + """Stream raw PGN text blocks, one game at a time, without loading + the whole file -- these corpora run into the GB range (twic.pgn + alone is 3.5GB).""" + with open(pgn_path, "r", encoding="utf-8", errors="replace") as f: + header_lines = [] + movetext_lines = [] + in_headers = True + for line in f: + if line.startswith("["): + if not in_headers and (header_lines or movetext_lines): + yield "".join(header_lines) + "\n" + "".join(movetext_lines) + header_lines, movetext_lines = [], [] + header_lines.append(line) + in_headers = True + elif line.strip() == "": + if header_lines and not movetext_lines: + in_headers = False + else: + in_headers = False + movetext_lines.append(line) + if header_lines or movetext_lines: + yield "".join(header_lines) + "\n" + "".join(movetext_lines) + + +def normalized_fen_key(fen): + """Board+side-to-move+castling+ep, dropping halfmove/fullmove + counters -- used only for dedup accounting, never written out.""" + parts = fen.split() + return " ".join(parts[:4]) + + +def process_pgn(path, min_elo, min_ply, stats, seen_keys, out_f): + kept_games = 0 + scanned_games = 0 + for raw in iter_game_texts(path): + scanned_games += 1 + game = chess.pgn.read_game(io.StringIO(raw)) + if game is None: + continue + headers = game.headers + try: + we = int(headers.get("WhiteElo", "0")) + be = int(headers.get("BlackElo", "0")) + except ValueError: + we = be = 0 + if min_elo and (we < min_elo or be < min_elo): + stats["rejected_elo"] += 1 + continue + + moves = list(game.mainline_moves()) + if len(moves) < min_ply: + stats["rejected_short"] += 1 + continue + + result = headers.get("Result", "*") + stats["games_by_result"][result] += 1 + stats["source_games"][path.name] += 1 + kept_games += 1 + + board = game.board() + move_idx = 0 + boundaries = [round(f * len(moves)) for f in FRACTIONS] + # de-dup consecutive identical boundaries (very short-but-kept + # games can round two fractions to the same ply) + seen_boundaries = set() + bi = 0 + for target_ply in boundaries: + target_ply = max(1, min(target_ply, len(moves) - 1)) + while move_idx < target_ply: + board.push(moves[move_idx]) + move_idx += 1 + stage = STAGE_NAMES[bi] + bi += 1 + if target_ply in seen_boundaries: + stats["skipped_duplicate_boundary"] += 1 + continue + seen_boundaries.add(target_ply) + if board.is_game_over(): + stats["skipped_game_over"] += 1 + continue + fen = board.fen() + key = normalized_fen_key(fen) + stats["stage_total"][stage] += 1 + stats["material_bucket"][material_bucket(board)] += 1 + stats["side_to_move"]["white" if board.turn else "black"] += 1 + if key in seen_keys: + stats["duplicate_fens"] += 1 + stats["stage_duplicates"][stage] += 1 + continue + seen_keys.add(key) + out_f.write(f"setboard {fen}\n") + out_f.write("go\n") + stats["written"] += 1 + stats["stage_written"][stage] += 1 + + stats["scanned_games"] += scanned_games + stats["kept_games"] += kept_games + return kept_games + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("pgn_paths", nargs="+") + ap.add_argument("--min-elo", type=int, default=2000) + ap.add_argument("--min-ply", type=int, default=30, + help="minimum game length in plies to be sampled " + "at all -- filters short draws/miniatures") + ap.add_argument("--out", required=True) + ap.add_argument("--progress-every", type=int, default=50000) + args = ap.parse_args() + + stats = { + "scanned_games": 0, + "kept_games": 0, + "rejected_elo": 0, + "rejected_short": 0, + "skipped_game_over": 0, + "skipped_duplicate_boundary": 0, + "duplicate_fens": 0, + "written": 0, + "games_by_result": collections.Counter(), + "source_games": collections.Counter(), + "stage_total": collections.Counter(), + "stage_written": collections.Counter(), + "stage_duplicates": collections.Counter(), + "material_bucket": collections.Counter(), + "side_to_move": collections.Counter(), + } + seen_keys = set() + + with open(args.out, "w") as out_f: + for p in args.pgn_paths: + path = Path(p) + print(f"processing {path.name}...", file=sys.stderr) + before = stats["kept_games"] + process_pgn(path, args.min_elo, args.min_ply, stats, seen_keys, out_f) + print(f" kept {stats['kept_games'] - before} games " + f"(running total written: {stats['written']})", file=sys.stderr) + + print("\n=== summary ===") + print(f"games scanned: {stats['scanned_games']:,}") + print(f"games kept: {stats['kept_games']:,}") + print(f" rejected (elo): {stats['rejected_elo']:,}") + print(f" rejected (too short): {stats['rejected_short']:,}") + print(f"positions written: {stats['written']:,}") + print(f"duplicate fens skipped: {stats['duplicate_fens']:,} " + f"({100.0*stats['duplicate_fens']/max(1,stats['written']+stats['duplicate_fens']):.1f}%)") + print(f"game-over boundaries skipped: {stats['skipped_game_over']:,}") + print() + print("by result:") + for r, c in stats["games_by_result"].most_common(): + print(f" {r:>8}: {c:,}") + print() + print("by source file:") + for s, c in stats["source_games"].most_common(): + print(f" {s}: {c:,}") + print() + print("by game stage (written / total-before-dedup):") + for stage in STAGE_NAMES: + w = stats["stage_written"][stage] + t = stats["stage_total"][stage] + d = stats["stage_duplicates"][stage] + print(f" {stage:>10}: written={w:,} dup_rate={100.0*d/max(1,t):.1f}%") + print() + print("by material bucket (0=bare kings .. 7=full army):") + for b in range(8): + print(f" bucket {b}: {stats['material_bucket'][b]:,}") + print() + print("by side to move:") + for k, c in stats["side_to_move"].items(): + print(f" {k}: {c:,}") + + +if __name__ == "__main__": + main() diff --git a/src/eval_tune/compare_vs_crafty.py b/src/eval_tune/compare_vs_crafty.py new file mode 100644 index 0000000..318ff87 --- /dev/null +++ b/src/eval_tune/compare_vs_crafty.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python3 +""" +Cross-engine static-eval audit: compare typhoon's `eval` command output +against a *crowd* of other hand-crafted-eval engines (Crafty, Kiwi, Fruit, +GNU Chess -- all pre-NNUE, additive-term evals, so genuinely comparable to +typhoon's own, unlike Stockfish's NNUE), rank the positions where typhoon +is the outlier against the crowd's median, and attribute the gap back to +typhoon's own named EVAL_TERM lines (from a build with EVAL_DUMP=1). + +Methodology, and the gotchas that cost real time to find (2026-09-2x +eval audit session): + + - Build typhoon with EVAL_DUMP=1 (see CLAUDE.md's build instructions) + to get the per-term breakdown table, not just the "Static eval:" + total. The breakdown itself is a SEPARATE command, `dump eval` + (EvalTraceReport() in testeval.c) -- it reports on the terms + recorded by the *last* Eval() call, so the script sends `eval` then + `dump eval` back to back for every position, not `eval` alone. + + - Run typhoon with --hash none. Not an optimization -- a correctness- + adjacent perf fix. SetBoardCommand -> PreGameReset(FALSE) -> + ClearHashTable() unconditionally, so every single `setboard` with a + real --hash size does a full memset of the whole hash table. Over + 2000 positions this was the entire difference between ~46s and + ~2.2s (21x) for byte-identical output. Always pass --hash none for + eval-only batch runs; there is no search happening, so there is + nothing to hash. + + - Feed positions via `script <file>` (one `setboard <fen>` / `eval` + pair per line), not a single giant --command string. A few thousand + positions' worth of inline command text can exceed the shell's/ + kernel's argv length limits; `script` reads from a file instead and + has no such ceiling. + + - Only ONE process per engine for the whole batch -- every engine here + correctly returns different, real scores across repeated + setboard/eval calls in a single session; none needs restarting per + position. (Verified directly this session by feeding + /usr/local/bin/crafty three visibly different positions in one + stdin session and getting three different totals.) + + - Use /usr/local/bin/crafty (whichever `crafty` resolves to on PATH), + NOT ~/programming/chess/crafty-25.2/crafty (formerly ~/crafty/crafty + -- that whole directory was moved to this path). The latter is a + different, older build (confirmed via md5) that a sibling script + here (compare_king_safety_crafty.py) found DOES silently reuse the + first position's cached score on a second setboard+score in one + session -- that bug is specific to that binary, not a general + Crafty limitation, and does not reproduce against + /usr/local/bin/crafty. If you ever see suspiciously identical + Crafty totals across different positions, check which crafty + binary you're actually calling before assuming a new engine bug. + + - Crowd engines each have their own sign convention for the "eval" + command's printed number, confirmed empirically (setboard a position + with white down a whole knight and check the sign/magnitude), not + assumed from reading their source alone: + * crafty `score`, kiwi `eval` -- print the score from WHITE's + perspective already, no flip needed. + * fruit `eval` -- prints from the SIDE-TO-MOVE's perspective + (negamax convention: eval.cpp's `eval()` explicitly negates for + Black at the very end). Must flip sign when it's Black to move + to get a White-perspective number comparable to the others. + * gnuchess (gnuchessr) `score` -- ALWAYS prints "(white to + move)" and an always-White-relative number, regardless of the + loaded position's actual active color. Confirmed empirically: + loading the identical position via two xpos files differing + only in trailing "w"/"b" produced byte-identical output both + times. Root cause (read, not guessed): `xget` -> GetXGame() + (dspcom.c) sets the `computer` global from the file's color + field but never touches the separate `player` global that + ScorePosition()'s perspective and the printed "(%s to move)" + label actually key off -- `player` is a zero-initialized global + that only search.c's search loop ever updates, so outside of an + actual search it just stays at its BSS-zero value (White). Net + effect: harmless for this script (means no stm-relative flip is + ever needed for gnuchess) but it DOES mean gnuchess's score + never reflects tempo/mobility-from-whoever's-actually-on-move + nuance -- a known accuracy ceiling on this crowd member, not a + bug in this script. + Getting a sign convention wrong silently halves the apparent + disagreement rate (a sign-flipped engine agrees with the crowd + exactly when it should maximally disagree, and vice versa) rather + than crashing, so it does not announce itself -- always spot check + a lopsided material position per engine before trusting a new one's + numbers. + + - Kiwi's own `kiwi` binary in ~/programming/chess/kiwi can be stale + relative to its own source (confirmed: mtimes showed the checked-in + binary predated winboard.cxx/engine.cxx, and it silently mis-parses + `eval` as an illegal `usermove` rather than erroring, because + command-table dispatch on prefix match falls through to + cmd_OpponentMoves on a lookup miss) -- there was already a correctly + rebuilt `kiwi.new` sitting alongside it in that same session, which + is what this script actually points at + (KIWI_BIN, below). If kiwi's numbers ever look wrong, rebuild with + `env OBJDIR=. LD=clang++ CC=clang++ gmake` from ~/programming/chess/kiwi + (its Makefile uses GNU-make-only `$(addprefix ...)`, so it must be + `gmake`, not BSD `make`; g++ doesn't exist on this box, hence the + clang++ overrides) and re-point KIWI_BIN at the fresh binary. + + - gnuchess (gnuchessr, ~/programming/chess/gnuchess/src) has no + `setboard`/`position fen` command at all -- the only way to load an + arbitrary position is `xget`, an interactive prompt that reads a + FILE in gnuchess's old xboard.position.read format (just FEN's + piece-placement and active-color fields, one line, no + castling/ep/clock -- castling rights are simply lost, an inherent + approximation for this crowd member). `_build_gnuchess_script` + writes one such tiny file per position under + /tmp/typhoon/eval_audit/gnuchess/ rather than trying to reuse one + filename, since the position must be fully written before the + single stdin-scripted process reaches that `xget` line. Also: the + process needs one leading blank line to satisfy its startup + "Enter [moves] minutes[:sec]..." time-control prompt before the + "Your move is?" loop (where `xget`/`score`/`quit` are valid + commands) even begins reading commands -- a script missing that + blank line gets its first real command silently swallowed by the + time-control prompt instead. Also needs its OWN directory as cwd + (same gotcha class as Glaurung/kpk.bin in an earlier version of + this crowd): run from elsewhere it prints "NO LANGFILE (file + gnuchess.lang not found)" and exits immediately instead of falling + back gracefully. + + - Compare typhoon against the crowd's MEDIAN, not vs. any single + other engine and not vs. the mean. Median is robust to any one + crowd engine being the actual outlier on a given position (e.g. a + known-narrow specialization one of them has and the others don't); + a mean lets one bad engine drag the "consensus" toward itself. + Historically (see the old Crafty-only version of this script) + averaging in an NNUE engine (Stockfish) made typhoon look like the + outlier for tracking a hand-crafted eval correctly (9 of 10 checked + outliers were actually typhoon+Crafty agreeing within ~0.3 pawns + with Stockfish alone ~12-15 pawns off) -- same failure mode extends + to any structurally-different evaluator diluting the crowd, so keep + the crowd restricted to same-family (additive, hand-tuned) evals. + + - This box's kern.ipc.semmni (max SysV semaphore sets) is only 50. + Every typhoon process leaks its semaphore set on `kill -9` (a clean + `quit`/EOF releases it; a hard kill does not, regardless of the + unix.c pthread-mutex/condvar rewrite -- that fixed a different + leak). Repeatedly killing hung/timed-out typhoon processes during + debugging can silently exhaust the limit; once exhausted, EVERY new + typhoon process falls back to a ~1000x-slower 100ms polling loop + for input, which looks exactly like a hang or a new performance + regression. Check with `ipcs -s | grep $(whoami) | wc -l` before + concluding a slowdown is a code bug; clean up with `ipcrm -s <id>` + for each leaked entry (or `for id in $(ipcs -s | grep $(whoami) | + awk '{print $2}'); do ipcrm -s "$id"; done`). + +Usage: + + python3 compare_vs_crafty.py <fen_source> [--limit N] [--top K] + [--engines crafty,kiwi,fruit,gnuchess] + + <fen_source> is either a path to an .ep_-style file (lines of + "setboard <fen>" / "go", as in ../tests/*.ep_) or a plain file of one + FEN per line. Requires a typhoon binary built with EVAL_DUMP=1 at + ../typhoon (build it first: `gmake clean && gmake -j5 GENETIC=1 + PERF_COUNTERS=1 MP=1 SIXTYFOUR=1 EVAL_DUMP=1` from src/), plus + whichever crowd engine binaries are selected (all on by default; + see ENGINE_SPECS for their expected paths). + +Output: prints the top-K disagreements (|typhoon(white) - +crowd_median(white)|), each crowd engine's own white-perspective score +for context, and typhoon's largest contributing EVAL_TERM line(s) from +its own perspective -- a quick first read on which term is driving the +gap, before digging in by hand. +""" +import argparse +import re +import statistics +import subprocess +import sys +from pathlib import Path + +SRC_DIR = Path(__file__).resolve().parent.parent +TYPHOON_BIN = SRC_DIR / "typhoon" +SCRATCH = Path("/tmp/typhoon/eval_audit") + +CHESS_DIR = Path.home() / "programming" / "chess" + +# Every crowd engine speaks a different protocol/command for "set this +# position, print static eval." `kind` selects how the position+eval +# commands are framed; `parse_re` must capture exactly the numeric score +# (possibly negative) as group 1; `scale` converts its native units to +# pawns; `stm_relative` marks engines whose printed number is from the +# SIDE TO MOVE's perspective (needs a sign flip on Black-to-move +# positions) rather than already being White-relative. +ENGINE_SPECS = { + "crafty": dict( + binary="crafty", cwd=None, kind="xboard", + startup="ponder off\n", pos_fmt="setboard {fen}\n", eval_cmd="score", + parse_re=r'^total\.+\s+([+-]?\d+\.\d+)', scale=1.0, stm_relative=False, + ), + "kiwi": dict( + binary=str(CHESS_DIR / "kiwi" / "kiwi.new"), cwd=None, kind="xboard", + startup="", pos_fmt="setboard {fen}\n", eval_cmd="eval", + parse_re=r'^Static eval\s*=\s*([+-]?\d+)', scale=0.01, stm_relative=False, + ), + "fruit": dict( + binary=str(CHESS_DIR / "fruit" / "fruit"), cwd=None, kind="uci", + startup="", pos_fmt="position fen {fen}\n", eval_cmd="eval", + parse_re=r'info string eval\s+([+-]?\d+)', scale=0.01, stm_relative=True, + ), + "gnuchess": dict( + binary=str(CHESS_DIR / "gnuchess" / "src" / "gnuchessr"), + cwd=str(CHESS_DIR / "gnuchess" / "src"), + kind="gnuchess", scale=0.01, stm_relative=False, + # Not anchored at line-start: gnuchess's "Your move is? " prompt + # precedes "Static score..." on the SAME line, unlike every + # other engine here which prints its own line cleanly. + parse_re=r'Static score \(\w+ to move\):\s*([+-]?\d+)', + ), +} + + +def extract_fens(path: Path) -> list[str]: + fens = [] + for line in path.read_text().splitlines(): + line = line.strip() + if line.startswith("setboard "): + fens.append(line[len("setboard "):].strip()) + elif line and not line.startswith(("go", "#")) and " " in line and "/" in line: + # plain FEN file + fens.append(line) + return fens + + +def run_typhoon_eval_dump(fens: list[str]) -> str: + SCRATCH.mkdir(parents=True, exist_ok=True) + script_path = SCRATCH / "typhoon_script.txt" + out_path = SCRATCH / "typhoon_out.txt" + with open(script_path, "w") as f: + for fen in fens: + # `eval` alone only prints the "Static eval:" total; the + # per-term "--- WHITE ... BLACK" breakdown table comes from + # the separate `dump eval` command (EvalTraceReport(), + # testeval.c), which reports on the terms from the *last* + # Eval() call -- so it must follow `eval` for the same + # position, not stand alone. + f.write(f"setboard {fen}\neval\ndump eval\n") + cmd = [str(TYPHOON_BIN), "--cpus", "1", "--hash", "none", + "--logfile", "-", "--batch", "--command", + f"force; book name /nonexistent.book.bin; script {script_path}"] + with open(out_path, "w") as out: + subprocess.run(cmd, stdout=out, stderr=subprocess.STDOUT, cwd=SRC_DIR, + timeout=max(60, len(fens) // 20)) + return out_path.read_text() + + +def run_crowd_engine(name: str, fens: list[str]) -> str: + spec = ENGINE_SPECS[name] + if spec["kind"] == "gnuchess": + script = _build_gnuchess_script(fens) + else: + script = spec["startup"] + for fen in fens: + script += spec["pos_fmt"].format(fen=fen) + spec["eval_cmd"] + "\n" + script += "quit\n" + p = subprocess.run([spec["binary"]], input=script, capture_output=True, + text=True, cwd=spec["cwd"], timeout=max(60, len(fens) // 20)) + return p.stdout + + +def _build_gnuchess_script(fens: list[str]) -> str: + """gnuchess (gnuchessr) has no `setboard`/`position fen` -- the only + way to load an arbitrary position is `xget`, which prompts for a + filename and reads a file in its own old xboard.position.read + format: just FEN's first two fields (piece placement + active + color), one per file, no castling/ep/clock info. Also: a leading + blank line is required first to accept the default time control + prompt before the "Your move is?" command loop (where `xget` is a + valid command) even starts accepting input.""" + gc_dir = SCRATCH / "gnuchess" + gc_dir.mkdir(parents=True, exist_ok=True) + script = "\n" + for i, fen in enumerate(fens): + parts = fen.split() + pos_path = gc_dir / f"pos_{i}.xpos" + pos_path.write_text(f"{parts[0]} {parts[1]}\n") + script += f"xget\n{pos_path}\nscore\n" + script += "quit\n" + return script + + +def parse_typhoon(text: str, n: int): + evals = [float(x) for x in + re.findall(r'^Static eval:\s*([+-]?\d+\.\d+)', text, re.M)] + assert len(evals) == n, f"expected {n} typhoon evals, got {len(evals)}" + return evals + + +def parse_crowd_engine(name: str, text: str, stm: list[str], n: int): + """Returns n scores in pawns, from WHITE's perspective.""" + spec = ENGINE_SPECS[name] + raw = [float(x) for x in re.findall(spec["parse_re"], text, re.M)] + assert len(raw) == n, ( + f"expected {n} {name} evals, got {len(raw)} -- protocol/parse_re " + f"mismatch, or {name}'s binary crashed/changed output format") + white = [] + for val, side in zip(raw, stm): + v = val * spec["scale"] + if spec["stm_relative"] and side == 'b': + v = -v + white.append(v) + return white + + +LABEL_RE_CACHE = {} + + +def term_labels_from_eval_c() -> list[str]: + """Pull every EVAL_TERM(...) call's label string directly out of + eval.c, so this stays in sync with the source without needing a + separately-maintained list.""" + src = (SRC_DIR / "eval.c").read_text() + labels = set() + for m in re.finditer(r'EVAL_TERM\(', src): + start = m.end() + depth = 1 + i = start + while depth > 0 and i < len(src): + if src[i] == '(': + depth += 1 + elif src[i] == ')': + depth -= 1 + i += 1 + # Allow escaped quotes/backslashes inside the string literal + # (e.g. eval.c's own "in/out of \"action\"" label) -- a naive + # [^"]* stops at the first escaped quote and can pick up a + # spurious empty trailing match instead of the real label. + strs = re.findall(r'"((?:[^"\\]|\\.)*)"', src[start:i - 1]) + if strs: + labels.add(strs[-1].replace('\\"', '"')) + return sorted(labels, key=len, reverse=True) + + +def get_label_re(): + if 're' not in LABEL_RE_CACHE: + labels = term_labels_from_eval_c() + LABEL_RE_CACHE['re'] = re.compile( + '(' + '|'.join(re.escape(l) for l in labels) + r')\s*(-|[a-h][1-8])?\s+(-?\d+)') + return LABEL_RE_CACHE['re'] + + +def parse_terms_for_block(block: str): + """block is the text between '--- WHITE' and 'Static eval:' for one + position. Returns {label: contribution_to_WHITE} (white/black columns + split at column 40, matching the dump's fixed layout). Always + White-relative regardless of side to move -- each column is that + color's own raw contribution, so White's column adds and Black's + subtracts from the White-relative total, independent of stm. (Do not + key the sign off stm: that gives the contribution to the side TO + MOVE instead, which flips sign on every Black-to-move position and + silently destroys any correlation against a White-relative residual.) + """ + label_re = get_label_re() + contrib = {} + for line in block.splitlines(): + for half, color in ((line[:40], 'w'), (line[40:], 'b')): + for m in label_re.finditer(half): + label, val = m.group(1), int(m.group(3)) + sign = 1 if color == 'w' else -1 + contrib[label] = contrib.get(label, 0) + sign * val + return contrib + + +def split_typhoon_blocks(text: str) -> list[str]: + # Each block runs from its own "--- WHITE" header up to (but not + # including) the NEXT position's leading board-dump text -- trim at + # this block's own "Static eval:" line, or the raw split would + # include the following position's FEN/hash/board-art/"Material:" + # preamble as trailing garbage, which can spuriously match the term + # regex (large hex-signature/material digit runs sitting near label + # text by coincidence). + parts = re.split(r'(?=^--- WHITE)', text, flags=re.M) + blocks = [] + for p in parts: + if not p.startswith('--- WHITE'): + continue + m = re.search(r'^Static eval:.*$', p, flags=re.M) + blocks.append(p[:m.end()] if m else p) + return blocks + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("fen_source") + ap.add_argument("--limit", type=int, default=2000, + help="max positions to sample (default 2000)") + ap.add_argument("--top", type=int, default=20, + help="how many biggest disagreements to print") + ap.add_argument("--engines", default=",".join(ENGINE_SPECS), + help=f"comma-separated crowd engines to use, from " + f"{{{','.join(ENGINE_SPECS)}}} (default: all)") + ap.add_argument("--sign-deadband", type=float, default=0.5, + help="pawns; both |typhoon| and |crowd median| (or " + "unanimous crowd value) must clear this to count " + "as a real sign disagreement, not ~0 noise " + "(default 0.5)") + ap.add_argument("--spread-floor", type=float, default=0.25, + help="pawns; floor on crowd stdev used when ranking " + "gap/spread so a near-unanimous crowd doesn't " + "produce a division-by-near-zero blowup " + "(default 0.25)") + args = ap.parse_args() + + engines = [e.strip() for e in args.engines.split(",") if e.strip()] + for e in engines: + if e not in ENGINE_SPECS: + sys.exit(f"unknown engine {e!r} -- choose from {list(ENGINE_SPECS)}") + if not Path(ENGINE_SPECS[e]["binary"]).is_absolute() and \ + ENGINE_SPECS[e]["cwd"] is None: + continue # resolved via PATH (e.g. crafty) + if not TYPHOON_BIN.exists(): + sys.exit(f"missing {TYPHOON_BIN} -- build with EVAL_DUMP=1 first " + "(see this file's docstring)") + + fens = extract_fens(Path(args.fen_source))[:args.limit] + stm = [f.split()[1] for f in fens] + print(f"{len(fens)} positions, crowd = {engines}", file=sys.stderr) + + ty_text = run_typhoon_eval_dump(fens) + ty_evals = parse_typhoon(ty_text, len(fens)) + ty_white = [e if s == 'w' else -e for e, s in zip(ty_evals, stm)] + + crowd_white = {} + for name in engines: + text = run_crowd_engine(name, fens) + crowd_white[name] = parse_crowd_engine(name, text, stm, len(fens)) + + blocks = split_typhoon_blocks(ty_text) + assert len(blocks) == len(fens), ( + f"{len(blocks)} eval-dump blocks vs {len(fens)} positions -- " + "did the typhoon binary actually have EVAL_DUMP=1?") + contribs = [parse_terms_for_block(b) for b in blocks] + + n = len(fens) + per_pos = [] # (median, stdev, residual, is_extreme, extreme_dir) + for i in range(n): + crowd_vals = [crowd_white[name][i] for name in engines] + median = statistics.median(crowd_vals) + stdev = statistics.pstdev(crowd_vals) if len(crowd_vals) > 1 else 0.0 + residual = ty_white[i] - median + all_vals = crowd_vals + [ty_white[i]] + is_max = ty_white[i] == max(all_vals) and all_vals.count(ty_white[i]) == 1 + is_min = ty_white[i] == min(all_vals) and all_vals.count(ty_white[i]) == 1 + per_pos.append((median, stdev, residual, is_max, is_min)) + + def print_position(i, extra=""): + median = per_pos[i][0] + crowd_str = " ".join(f"{name}={crowd_white[name][i]:+6.2f}" + for name in engines) + print(f"{extra}ty(w)={ty_white[i]:+6.2f} median(w)={median:+6.2f} " + f"{crowd_str} {fens[i]}") + top_terms = sorted(contribs[i].items(), key=lambda kv: -abs(kv[1]))[:4] + for label, val in top_terms: + print(f" {label:38s} {val:+5d}") + + # --- 1. Raw magnitude disagreements (original behavior) ------------- + diffs = sorted(range(n), key=lambda i: -abs(per_pos[i][2])) + print(f"\n=== Top {args.top} disagreements " + f"(|typhoon(white) - crowd_median(white)|) ===\n") + for i in diffs[:args.top]: + print_position(i, extra=f"diff={per_pos[i][2]:+7.2f} ") + + # --- 2. Gap normalized by crowd spread ------------------------------- + # A raw pawn gap conflates "typhoon is wrong" with "this position is + # just wild and the whole crowd disagrees with itself." Dividing by + # the crowd's own stdev (floored, so a near-unanimous crowd doesn't + # blow up the ratio) surfaces cases where typhoon breaks from an + # otherwise-tight consensus, even by a modest raw amount. + spread_ranked = sorted( + range(n), key=lambda i: -abs(per_pos[i][2]) / max(per_pos[i][1], args.spread_floor)) + print(f"\n=== Top {args.top} disagreements normalized by crowd spread " + f"(|residual| / max(crowd_stdev, {args.spread_floor})) ===\n") + for i in spread_ranked[:args.top]: + ratio = abs(per_pos[i][2]) / max(per_pos[i][1], args.spread_floor) + print_position(i, extra=f"ratio={ratio:6.2f} stdev={per_pos[i][1]:5.2f} ") + + # --- 3. Typhoon as the lone extreme value ---------------------------- + extremes = [i for i in range(n) if per_pos[i][3] or per_pos[i][4]] + extremes.sort(key=lambda i: -abs(per_pos[i][2])) + print(f"\n=== Positions where typhoon is the strict min or max of " + f"typhoon+crowd ({len(extremes)} of {n}) ===\n") + for i in extremes[:args.top]: + direction = "HIGHEST" if per_pos[i][3] else "LOWEST" + print_position(i, extra=f"typhoon is {direction} ") + + # --- 4. Sign disagreements ------------------------------------------- + db = args.sign_deadband + vs_median, vs_unanimous = [], [] + for i in range(n): + ty, median = ty_white[i], per_pos[i][0] + if abs(ty) >= db and abs(median) >= db and (ty > 0) != (median > 0): + vs_median.append(i) + crowd_vals = [crowd_white[name][i] for name in engines] + if all(abs(v) >= db for v in crowd_vals) and \ + len({v > 0 for v in crowd_vals}) == 1: + crowd_sign_positive = crowd_vals[0] > 0 + if abs(ty) >= db and (ty > 0) != crowd_sign_positive: + vs_unanimous.append(i) + print(f"\n=== Typhoon disagrees with crowd MEDIAN's sign " + f"(both sides >= {db} pawns) -- {len(vs_median)} positions ===\n") + for i in vs_median[:args.top]: + print_position(i) + print(f"\n=== Typhoon disagrees with a UNANIMOUS crowd's sign " + f"(every engine agrees, typhoon alone doesn't; >= {db} pawns) " + f"-- {len(vs_unanimous)} positions ===\n") + for i in vs_unanimous[:args.top]: + print_position(i) + if "gnuchess" in engines and len(engines) > 1: + others = [e for e in engines if e != "gnuchess"] + vs_unanimous_no_gc = [] + for i in range(n): + vals = [crowd_white[name][i] for name in others] + if all(abs(v) >= db for v in vals) and len({v > 0 for v in vals}) == 1: + if abs(ty_white[i]) >= db and (ty_white[i] > 0) != (vals[0] > 0): + vs_unanimous_no_gc.append(i) + print(f"\n (gnuchess drops castling rights and ignores stm; " + f"same check EXCLUDING gnuchess: " + f"{len(vs_unanimous_no_gc)} positions)") + + # --- 5. Per-term correlation with the White-relative residual -------- + # For each EVAL_TERM label, correlate its own signed White-relative + # contribution against ty_white - crowd_median across the whole + # sample. A term whose contribution consistently tracks the residual + # is a candidate for being over- or under-weighted relative to the + # crowd's implicit consensus; a term with no correlation isn't + # driving the disagreement even if it's large in any one position. + residuals = [p[2] for p in per_pos] + all_labels = set() + for c in contribs: + all_labels.update(c.keys()) + term_stats = [] + for label in all_labels: + vals = [contribs[i].get(label, 0) for i in range(n)] + if len(set(vals)) < 2: + continue # constant (e.g. always zero) -- correlation undefined + try: + corr = statistics.correlation(vals, residuals) + except statistics.StatisticsError: + continue + mean_abs = sum(abs(v) for v in vals) / n + term_stats.append((label, corr, mean_abs, corr * mean_abs)) + term_stats.sort(key=lambda t: -abs(t[3])) + print(f"\n=== EVAL_TERM correlation with (typhoon - crowd_median), " + f"whole sample (n={n}) ===") + print("(positive corr = term grows typhoon's score in the same " + "direction as its disagreement with the crowd -- a candidate " + "for being over-weighted; negative = under-weighted)\n") + for label, corr, mean_abs, score in term_stats[:20]: + print(f" {label:38s} corr={corr:+.3f} mean|contrib|={mean_abs:6.2f} " + f"score={score:+.3f}") + + # --- 6. Global directional bias --------------------------------------- + mean_r = statistics.mean(residuals) + median_r = statistics.median(residuals) + w_res = [residuals[i] for i in range(n) if stm[i] == 'w'] + b_res = [residuals[i] for i in range(n) if stm[i] == 'b'] + print(f"\n=== Global bias: typhoon(white) - crowd_median(white) ===\n") + print(f" overall: mean={mean_r:+.3f} median={median_r:+.3f} n={n}") + if w_res: + print(f" white to move: mean={statistics.mean(w_res):+.3f} " + f"median={statistics.median(w_res):+.3f} n={len(w_res)}") + if b_res: + print(f" black to move: mean={statistics.mean(b_res):+.3f} " + f"median={statistics.median(b_res):+.3f} n={len(b_res)}") + print(" (a nonzero overall mean/median with the same sign on " + "both stm splits points at a constant-ish term -- tempo, draw " + "score, a global scale factor -- rather than a positional one)") + + # --- 7. Residual by material phase ------------------------------------ + # Cheap, self-contained phase proxy from the FEN alone (not typhoon's + # own internal phase bucketing) so this works even without EVAL_DUMP + # internals: sum of N=3/B=3/R=5/Q=9 for both sides, pawns/kings + # excluded. Eval bugs are usually phase-local, and a flat top-K list + # hides that. + piece_val = {'n': 3, 'b': 3, 'r': 5, 'q': 9} + phases = [] + for fen in fens: + board = fen.split()[0] + phases.append(sum(piece_val.get(ch.lower(), 0) for ch in board if ch.isalpha())) + buckets = [(0, 12, "endgame (<=12)"), (13, 26, "middlegame (13-26)"), + (27, 999, "opening/near-full (>=27)")] + print(f"\n=== Residual by material phase (non-pawn point count, both sides) ===\n") + for lo, hi, name in buckets: + idxs = [i for i in range(n) if lo <= phases[i] <= hi] + if not idxs: + continue + r = [residuals[i] for i in idxs] + print(f" {name:28s} mean={statistics.mean(r):+.3f} " + f"median={statistics.median(r):+.3f} n={len(idxs)}") + + +if __name__ == "__main__": + main() diff --git a/src/eval_tune/match_play.py b/src/eval_tune/match_play.py index 5033cad..34a7917 100755 --- a/src/eval_tune/match_play.py +++ b/src/eval_tune/match_play.py @@ -96,7 +96,13 @@ _FIRST_MOVE_BETA_RE = re.compile( # count, (4) the rest of the line: the SAN PV text (WalkPV, util.c), or # "FL --" for a fail-low line (no PV -- the move that failed low isn't # necessarily the move eventually played) or "<SAN> ++" for a fail-high. -_ITER_RE = re.compile(r"^\s*(\d+)[+-]?\s+(-?\d+)\s+\S+\s+(\d+)\s*(.*)$") +# 2026-10-17: score group widened from (-?\d+) to ([+-]?\d+) -- typhoon +# itself never prints a leading "+" on a non-negative score (bare +# digits), but a UCI engine driven through PolyGlot's xboard adapter +# (e.g. Stockfish, see stockfish_xboard) DOES print an explicit "+" +# ("23 +28 198 700479 e4 c5"), which the old pattern rejected outright +# -- confirmed via direct regex test before this fix, not assumed. +_ITER_RE = re.compile(r"^\s*(\d+)[+-]?\s+([+-]?\d+)\s+\S+\s+(\d+)\s*(.*)$") # Reading PostMoveSearchReport's trailing block used to stop either at a # bounded line count or at gamelist.c's "_white(move N)_"/"_black(move N)_" # prompt -- both unreliable: the block's line count depends on whether the @@ -307,6 +313,13 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, white_stats = EngineStats() black_stats = EngineStats() white_last_fmb = black_last_fmb = None + # Tracked across the whole engine-interaction block below so the + # BrokenPipeError handler (see try/except wrapping it) can report + # which side/ply was in flight even when the pipe dies on a write + # that isn't inside the per-ply move loop (e.g. one of the startup + # sends, or a time-control send before the first ply). + ply_idx = -1 + white_to_move = None # Default Engine() spawns with NO --hash/--cpus at all, which means # a tiny 65536-entry hash table (main.c's memset-zero default) -- # far smaller than the 256m used everywhere else in this project's @@ -342,7 +355,8 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, # and the default behavior of a harness in active use should not # change silently underneath a running experiment. startup = ["book name /none", - "set ThinkOnOpponentsTime false"] + "set ThinkOnOpponentsTime false", + "set PostThrottle 0.0"] if post: startup.append("post") for command in startup: @@ -416,6 +430,7 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, tail = im.group(4).strip() if (last_iter_depth is not None and last_iter_nodes is not None and + last_iter_nodes > 0 and iter_nodes > last_iter_nodes and iter_depth > last_iter_depth): iter_log_ratio_sum += (math.log(iter_nodes / last_iter_nodes) @@ -434,10 +449,12 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, last_eval_score = iter_score last_eval_pv = tail[:-3].rstrip() if tail.endswith(" ++") else tail + abort_reason = None for _ in range(400): line = mover.stdout.readline() if not line: _dbg(tag, "EOF while scanning for move line") + abort_reason = "eof" break _dbg(tag, f"read(move-scan): {line!r}") _note_iter(line) @@ -445,15 +462,67 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, # Engine resigned instead of moving -- root.c prints # this and returns without ever printing "move ...", # so stop reading now rather than blocking forever. + # This IS a legitimate decisive result (unlike the + # other abort_reason cases below) -- resignation is a + # real chess outcome, not a harness/engine failure. + abort_reason = "resign" break m = _MOVE_RE.match(line) if m: - move_uci = m.group(1) + # root.c's "move ..." line is xboard/ICS-style + # (MoveToIcs, ics.c), which prints the promotion + # piece UPPERCASE ("e7e8Q") -- correct for that + # protocol, but chess.Move.from_uci() below enforces + # strict lowercase UCI and raises ValueError on it. + # Unlowered, EVERY promotion move got misclassified + # as illegal and the game aborted right as the + # promoting side (almost always the winning side in + # a long endgame) was about to seal the win -- this + # was the actual root cause of the ~47%-corrupted + # self-play corpus (docs/NNUE.md, 2026-10-17), not an + # engine stall/crash. Squares/files are already + # lowercase, so a blanket .lower() is safe here. + move_uci = m.group(1).lower() break + else: + # Read 400 lines without ever seeing "move ..." (and + # without EOF or resignation either) -- most likely an + # extremely verbose iteration output (deep search in a + # sparse endgame can print far more than 400 lines for a + # single move) exceeding this loop's hard cap while the + # engine is still alive and about to answer. + abort_reason = "line-cap-exceeded" if move_uci is None: - # side to move had nothing to play -- treat as loss for mover - result = 0.0 if white_to_move else 1.0 - return result, moves, move_evals, white_stats, black_stats + if abort_reason == "resign": + # side to move resigned -- legitimate loss for mover + result = 0.0 if white_to_move else 1.0 + return (result, moves, move_evals, white_stats, + black_stats, None) + # 2026-10-17: a real bug was found producing exactly this + # path for a non-resign reason (eof / line-cap) on a + # position where the mover was completely winning -- + # resignation was disabled for this corpus, so there is + # no legitimate explanation for move_uci staying None + # other than resign. The OLD code silently recorded this + # as an ordinary loss for the mover, which corrupted + # ~47% of a self-play training corpus with inverted + # results (see docs/NNUE.md). Never do that again: signal + # the caller with an aborted (result=None) game instead + # of guessing a winner, and capture enough context here + # to investigate later without needing to reproduce it + # live. + exit_code = mover.proc.poll() + abort_info = { + "reason": abort_reason, + "mover": "white" if white_to_move else "black", + "ply": ply_idx, + "fen_before_ply": board.fen(), + "mover_exit_code": exit_code, + "mover_alive": exit_code is None, + } + _dbg(tag, f"ABORT (not scored): {abort_info}") + return (None, moves, move_evals, white_stats, black_stats, + abort_info) # PostMoveSearchReport's stats block trails the "move" line # (confirmed against root.c and real output), not the other @@ -512,14 +581,32 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, else: black_last_fmb = ply_fmb + move = None try: move = chess.Move.from_uci(move_uci) - if move not in board.legal_moves: - result = 0.0 if white_to_move else 1.0 - return result, moves, move_evals, white_stats, black_stats + illegal = move not in board.legal_moves except ValueError: - result = 0.0 if white_to_move else 1.0 - return result, moves, move_evals, white_stats, black_stats + illegal = True + if illegal: + # The engine printed a "move ..." line, but it wasn't a + # legal move -- same reasoning as the move_uci-is-None + # case above: a real engine bug (producing garbage) + # deserves the same abort-not-score treatment, not a + # confidently-recorded loss with no diagnostic trail. + exit_code = mover.proc.poll() + abort_info = { + "reason": "illegal-move", + "mover": "white" if white_to_move else "black", + "ply": ply_idx, + "fen_before_ply": board.fen(), + "illegal_move_uci": move_uci, + "mover_exit_code": exit_code, + "mover_alive": exit_code is None, + } + _dbg(tag, f"ABORT (not scored): {abort_info}") + return (None, moves, move_evals, white_stats, black_stats, + abort_info) + assert move is not None # illegal is False only when the try succeeded board.push(move) moves.append(move_uci) score_white = None @@ -536,7 +623,7 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, result = 0.5 else: result = 1.0 if outcome.winner == chess.WHITE else 0.0 - return result, moves, move_evals, white_stats, black_stats + return result, moves, move_evals, white_stats, black_stats, None def _format_eval_comment(score_white, pv, depth): @@ -576,7 +663,15 @@ def build_pgn_game(start_fen, moves, result_white, white_name, black_name, game.headers["FEN"] = start_fen game.headers["Date"] = f"{date.today()}" game.setup(board) - result_str = {1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white] + # result_white is None for an ABORTED game (play_one_game's mover + # failed to produce a legal move for a non-resign reason -- see its + # abort_info return value) -- write a standard "unknown result" PGN + # tag rather than guessing a winner. fit_k_and_label.py's + # result_to_white_score() already treats "*" as "unknown, skip this + # game" for training purposes, so this alone keeps an aborted game + # out of any future training corpus without touching that script. + result_str = ({1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white] + if result_white is not None else "*") game.headers["Result"] = result_str node = game for i, move_uci in enumerate(moves): @@ -888,6 +983,48 @@ def main(): "Without this, local_ebf reads 0.000 -- the " "engines never print those lines because " "Engine.__init__ sends 'nopost'.") + ap.add_argument("--candidate-name", default="candidate", + help="PGN White/Black name recorded for the " + "candidate engine (default: 'candidate'). " + "2026-10-16: set this to something identifying " + "the eval/config actually used (e.g. " + "'candidate_nnue_loose') whenever the engine " + "isn't a plain HCE build -- self_play_games.pgn " + "is reused as NNUE training data later, and a " + "generic 'candidate' name makes it impossible " + "to tell which games came from a differently-" + "calibrated evaluator (whose scores shouldn't " + "be trusted as training labels) without " + "externally remembering run dates.") + ap.add_argument("--baseline-name", default="baseline", + help="PGN White/Black name recorded for the " + "baseline/head engine (default: 'baseline'); " + "same reasoning as --candidate-name.") + ap.add_argument("--no-color-swap", action="store_true", + help="play each sampled opening ONCE instead of " + "twice with colors swapped between head/" + "candidate. The swap exists to cancel out " + "color-dependent FIRST-MOVE ADVANTAGE in an " + "Elo-gating match; it is pure waste -- a byte-" + "for-byte duplicate game, just relabeled -- " + "whenever head_engine and candidate_engine are " + "the SAME binary (as every self-play " + "training-data run via " + "hce_training_corpus.sh is), because search " + "here is deterministic (--cpus 1) given a " + "fixed FEN+side-to-move, so swapping which " + "name is 'white'/'black' changes no move " + "actually played. Confirmed empirically " + "2026-10-16: consecutive game pairs in " + "nnue_self_play_training_data.pgn were exact " + "duplicates (identical FEN, identical move " + "list). Use this flag for self-play corpus " + "generation -- sample --games distinct " + "openings and play each once, rather than " + "--games/2 openings played twice -- and keep " + "the swap (the default) for real head-vs-" + "candidate Elo gating, where two different " + "binaries need it to cancel color bias.") ap.add_argument("--max-plies", type=int, default=200) ap.add_argument("--scratch", default="/usr/local/tmp/typhoon_match") ap.add_argument("--log", default="match_history.log") @@ -968,13 +1105,16 @@ def main(): head_engine = str(Path(args.head_engine).resolve()) candidate_engine = str(Path(args.candidate_engine).resolve()) - n_pairs = args.games // 2 - openings = sample_openings(args.pgn, n_pairs, seed=args.opening_seed) - n_games = len(openings) * 2 + if args.no_color_swap: + n_openings = args.games + else: + n_openings = args.games // 2 + openings = sample_openings(args.pgn, n_openings, seed=args.opening_seed) jobs = [] for fen in openings: - for candidate_is_white in (True, False): + seats = (True,) if args.no_color_swap else (True, False) + for candidate_is_white in seats: i = len(jobs) white_engine = candidate_engine if candidate_is_white else head_engine black_engine = head_engine if candidate_is_white else candidate_engine @@ -982,6 +1122,7 @@ def main(): candidate_points = 0.0 wins = draws = losses = 0 + aborted = 0 sprt = Sprt(args.elo0, args.elo1, args.alpha, args.beta) if args.sprt else None sprt_decision = None candidate_stats = EngineStats() @@ -1002,22 +1143,56 @@ def main(): if args.sd is None and head_time != cand_time: assert ((white_time == cand_time) == candidate_is_white), \ "time-odds seat mapping inverted" - result_white, moves, move_evals, white_stats, black_stats = play_one_game( - white_engine, black_engine, f"{args.scratch}/g{i}", fen, - depth=args.sd, - white_time_sec=white_time, black_time_sec=black_time, - max_plies=args.max_plies, post=args.post, - ) + try: + result_white, moves, move_evals, white_stats, black_stats, abort_info = ( + play_one_game( + white_engine, black_engine, f"{args.scratch}/g{i}", fen, + depth=args.sd, + white_time_sec=white_time, black_time_sec=black_time, + max_plies=args.max_plies, post=args.post, + ) + ) + except (BrokenPipeError, OSError) as exc: + # One of the two engine subprocesses died mid-game (crashed, + # was OOM-killed, etc.) -- play_one_game's own abort_info + # machinery only covers READ-side failures (EOF/line-cap + # while scanning for a move); a WRITE to an already-dead + # process's stdin raises BrokenPipeError straight out of + # Engine._send instead, uncaught anywhere below this. Left + # unhandled, that exception propagates out of this worker + # thread's run() through the Future and kills the entire + # match (every other in-flight/queued game lost) over a + # single flaky engine process -- confirmed live against a + # polyglot/Stockfish side via stockfish_xboard. Treat it the + # same way as any other abort: no result guessed, game + # skipped, run continues. + result_white = None + moves, move_evals = [], [] + white_stats, black_stats = EngineStats(), EngineStats() + abort_info = { + "reason": "engine-pipe-broken", + "exception": f"{type(exc).__name__}: {exc}", + } game_elapsed = time.monotonic() - game_start - candidate_result = result_white if candidate_is_white else (1.0 - result_white) - white_name = "candidate" if candidate_is_white else "baseline" - black_name = "baseline" if candidate_is_white else "candidate" + # result_white is None for an aborted game (play_one_game's mover + # failed to produce a legal move for a non-resign reason) -- do + # NOT guess a winner (see abort_info's own comment at the + # return sites in play_one_game for why this matters: it's + # exactly the bug that silently corrupted ~47% of an earlier + # self-play corpus). Propagate None through candidate_result too + # so run()'s caller can skip it rather than score it. + candidate_result = ( + None if result_white is None else + result_white if candidate_is_white else (1.0 - result_white) + ) + white_name = args.candidate_name if candidate_is_white else args.baseline_name + black_name = args.baseline_name if candidate_is_white else args.candidate_name game, _ = build_pgn_game(fen, moves, result_white, white_name, black_name, move_evals=move_evals) this_candidate_stats = white_stats if candidate_is_white else black_stats this_head_stats = black_stats if candidate_is_white else white_stats return (i, candidate_result, game, game_elapsed, - this_candidate_stats, this_head_stats) + this_candidate_stats, this_head_stats, abort_info) game_durations = [] job_iter = iter(jobs) @@ -1057,14 +1232,39 @@ def main(): finished, in_flight = wait(in_flight, return_when=FIRST_COMPLETED) for fut in finished: (i, candidate_result, game, game_elapsed, - this_candidate_stats, this_head_stats) = fut.result() + this_candidate_stats, this_head_stats, abort_info) = fut.result() + print(game, file=pgn_out, end="\n\n") + pgn_out.flush() + + if candidate_result is None: + # Aborted game (play_one_game's mover failed to + # produce a legal move for a non-resign reason -- + # see abort_info). Do NOT score it (no wins/ + # losses/draws/candidate_points/SPRT update, no + # contribution to the `done` denominator used for + # score/ETA) -- this is exactly the "silently + # record a decisive result with no diagnostic + # trail" bug that corrupted ~47% of an earlier + # self-play corpus (docs/NNUE.md, 2026-10-17). + # The game is still written to pgn_out above + # (Result "*", which fit_k_and_label.py's + # result_to_white_score() already treats as + # "unknown, skip" for training) so it isn't lost + # for post-mortem debugging, just not scored. + aborted += 1 + log.write(f"game {i}: ABORTED (not scored) " + f"abort_info={abort_info} " + f"elapsed={game_elapsed:.1f}s\n") + log.flush() + print(f" game {i}: ABORTED -- {abort_info}", + file=sys.stderr) + continue + done += 1 candidate_points += candidate_result game_durations.append(game_elapsed) candidate_stats.merge(this_candidate_stats) head_stats.merge(this_head_stats) - print(game, file=pgn_out, end="\n\n") - pgn_out.flush() if candidate_result == 1.0: wins += 1 elif candidate_result == 0.0: @@ -1131,7 +1331,7 @@ def main(): "(not waiting for in-flight games to finish naturally):", file=sys.stderr) print_final_summary(done, candidate_points, wins, draws, - losses, candidate_stats, head_stats, + losses, aborted, candidate_stats, head_stats, sprt, sprt_decision, args) # os._exit, not sys.exit: skips the `with` block's normal # ThreadPoolExecutor teardown (wait=True by default -- would @@ -1143,17 +1343,21 @@ def main(): # the file objects' own __exit__. os._exit(130) - print_final_summary(done, candidate_points, wins, draws, losses, + print_final_summary(done, candidate_points, wins, draws, losses, aborted, candidate_stats, head_stats, sprt, sprt_decision, args) -def print_final_summary(done, candidate_points, wins, draws, losses, +def print_final_summary(done, candidate_points, wins, draws, losses, aborted, candidate_stats, head_stats, sprt, sprt_decision, args): """Shared by the normal end-of-match path and the ^C handler below -- same CANDIDATE_SCORE/CANDIDATE_STATS/HEAD_STATS block either way, just - computed from whatever `done` games actually completed.""" + computed from whatever `done` games actually completed. `aborted` + (not included in `done` or in any win/draw/loss/score arithmetic) is + the count of games where a mover failed to produce a legal move for + a non-resign reason -- see play_one_game's abort_info. Surfaced here + so an elevated count is visible instead of silently discarded.""" n_games = done score = candidate_points / n_games if n_games else 0.5 elo, err = elo_diff(score, n_games) @@ -1182,10 +1386,11 @@ def print_final_summary(done, candidate_points, wins, draws, losses, f"cand={args.cand_time}s)") else: tc_str = f" TC=st{args.head_time}" + aborted_str = f" ABORTED={aborted}" if aborted else "" print(f"CANDIDATE_SCORE={score:.4f} GAMES={n_games} " f"WINS={wins} DRAWS={draws} LOSSES={losses} " f"ELO={elo:+.1f}+/-{err:.1f} LOWER95={lower95:.4f}" - f"{tc_str}{sprt_str}") + f"{tc_str}{sprt_str}{aborted_str}") print(f"CANDIDATE_STATS: {candidate_stats.summary()}") print(f"HEAD_STATS: {head_stats.summary()}") diff --git a/src/eval_tune/nnue_vs_head.sh b/src/eval_tune/nnue_vs_head.sh new file mode 100755 index 0000000..25a4c4f --- /dev/null +++ b/src/eval_tune/nnue_vs_head.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# +# Gated self-play match against a baseline binary. Historically this +# always compared the live working-tree build against head_reference/ +# -- generalized (2026-09-14) to take both binary paths as args so two +# arbitrary builds (e.g. a file-storm-only worktree build vs. the +# current working tree, to isolate one change at a time when several +# have landed together -- see king_safety_data/HANDOFF.md's session +# notes on not measuring two eval changes in one match) can be compared +# without hand-rolling a match_play.py invocation each time. +# +# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label] +# +# head_binary Baseline engine. Defaults to head_reference/typhoon +# (the usual "is this better than last checkpoint" +# question). +# candidate_binary Candidate engine. Defaults to a fresh copy of the +# live working-tree build (../typhoon) into +# /tmp/typhoon/typhoon_match_play_candidate -- pass +# an explicit path (already built) to skip that +# copy/freeze step entirely, e.g. when comparing +# two prebuilt worktree binaries. +# label Distinguishes scratch dir/log/pgn-out so multiple +# matches can run concurrently without colliding. +# Defaults to "match". +# +# ...and any further arguments are passed straight through to +# match_play.py, so a one-off flag no longer needs this file edited. +# +# Three env vars override the baked-in defaults (2026-10-03), so the +# common knobs are reachable without touching the invocation below: +# +# TC_ARGS time control. Default: "--st 4" +# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5" +# GAMES game cap. Default: 10000 +# +# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5" +# cannot resolve the effect sizes this project's LMR work actually +# produces. At the observed draw rate the standard error in Elo is about +# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000 +# games (~53h here) -- and every LMR match run to date has instead +# wandered around llr 0 for 2,000+ games and been abandoned, which is +# exactly what a true effect sitting BETWEEN the two hypotheses looks +# like. For a change you would only ship at >= 15 Elo, use +# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours +# instead of never. +# +# Examples: +# test_vs_head.sh # old default behavior +# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight +# +# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo: +# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh +# +# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the +# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in +# # THIS engine, which bounds what any depth-buying search change +# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity +# # rather than testing a hypothesis, and ELO=+/- prints either way. +# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo +# # of precision already separates "a ply is worth 25" from "worth 65". +# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \ +# test_vs_head.sh ../../head_reference/typhoon \ +# ../../head_reference/typhoon timeodds +# +# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/ +# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped +# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS" +# ahead of whatever args match_play.py's Engine class passes it -- +# that's the only change needed, since Engine spawns the given path +# directly via Popen (no shell), so a shebang script in its place works +# unmodified. HEAD is untouched, so it keeps running the classic HCE -- +# this is how you get an NNUE-candidate-vs-HCE-reference match without +# patching match_play.py's Engine (which currently passes identical +# engine_args to both sides): +# +# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh +# +# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by +# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/ +# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to +# also be set (the probe gate only does anything once NNUE eval itself +# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the +# same wrapper, enabling the lazy-eval accumulator probe gate at +# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE +# automatically, same convention as --nnueweights): +# +# NNUE_WEIGHTS=/path/to/net.nnue \ +# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \ +# test_vs_head.sh + +set -e + +# Only consume LEADING args that don't look like a flag as the three +# positionals (head/candidate/label) -- 2026-10-19 fix: a bare +# ./nnue_vs_head.sh --sd 10 used to land "--sd" in HEAD and "10" in +# CANDIDATE_ARG (since the old code unconditionally took "$1"/"$2"/"$3" +# regardless of content), the exact same footgun class the comment +# below already describes once biting the --shift-then-test-"$2" +# version of this script. A flag-looking first arg now means "no +# positionals given at all, everything is pass-through" instead. +POSITIONALS=() +while [ $# -gt 0 ] && [ "${#POSITIONALS[@]}" -lt 3 ]; do + case "$1" in + -*) break ;; + *) POSITIONALS+=("$1"); shift ;; + esac +done + +HEAD="${POSITIONALS[0]:-../../head_reference/typhoon}" +CANDIDATE_ARG="${POSITIONALS[1]:-}" +LABEL="${POSITIONALS[2]:-match}" +# Note the `-` (not `:-`) for the three env overrides below: an +# explicitly EMPTY value must be honored, not replaced by the default. +# SPRT_ARGS="" is the documented way to run without a hypothesis test +# (e.g. the time-odds calibration above), and `:-` would silently put +# --sprt back. +TC_ARGS="${TC_ARGS---st 4}" +SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}" +GAMES="${GAMES-10000}" + +# If the caller already passed --sd or --st through as an extra +# argument (e.g. ./nnue_vs_head.sh --sd 10), don't ALSO inject the +# baked-in TC_ARGS ("--st 4" by default) -- match_play.py's --sd/--st +# are mutually exclusive, so the two would collide +# ("error: argument --st: not allowed with argument --sd") even though +# the caller only asked for one of them. Pass-through wins; TC_ARGS is +# only the fallback when the caller didn't specify either. +for arg in "$@"; do + case "$arg" in + --sd|--st) TC_ARGS="" ;; + esac +done + +if [ -n "$CANDIDATE_ARG" ]; then + CANDIDATE="$CANDIDATE_ARG" +else + cd .. + gmake clean + gmake release MP=1 PERF_COUNTERS=1 + cd eval_tune + CANDIDATE=/tmp/typhoon/typhoon_match_play_candidate + cp -f ../typhoon_nnue ${CANDIDATE} + # typhoon_nnue's own TYPHOON_BIN/NNUE_WEIGHTS defaults are computed + # relative to ITS OWN location ("next to this script" -- see its + # header comment) -- correct when it's the real src/typhoon_nnue, + # but wrong once copied to /tmp/typhoon/ above: it would then look + # for the engine binary at /tmp/typhoon/typhoon and the weights + # file under /tmp/annotate_pgns/..., neither of which exist, and + # exit 1 immediately on every single spawn (2026-10-19: this is + # what produced hundreds of instant "BrokenPipeError" game aborts + # the first time this ran -- the copied wrapper was exiting before + # ever speaking the xboard protocol). match_play.py's Popen + # inherits this script's environment, so exporting the correct + # ABSOLUTE paths here -- computed from this script's own known-good + # location, not the relocated copy's -- fixes it without needing + # to touch typhoon_nnue itself. Only fill in NNUE_WEIGHTS if the + # caller didn't already set one explicitly. + export TYPHOON_BIN="$(cd .. && pwd)/typhoon" + : "${NNUE_WEIGHTS:=$(cd ../../annotate_pgns && pwd)/runs/sf98k/nnue_ckpt/nnue_weights.nnue}" + export NNUE_WEIGHTS +fi + +if [ -n "${NNUE_WEIGHTS:-}" ]; then + NNUE_WRAPPER=/tmp/typhoon/typhoon_match_play_candidate_nnue + NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\"" + if [ -n "${NNUE_PROBE_WEIGHTS:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\"" + fi + # NNUE_LAZY_MARGIN / NNUE_SUPERLAZY_MARGIN: 2026-10-16 looser- + # margin experiment (see docs/NNUE.md and command.c's "nnue + # lazymargin"/"nnue superlazyon"). NNUE_LAZY_MARGIN overrides every + # material bucket's lazy-probe margin with one flat cp value + # instead of the calibrated p98 table; NNUE_SUPERLAZY_MARGIN turns + # on the even-cheaper material-only pre-filter (off by default) + # with the given flat cp margin. Both require NNUE_WEIGHTS; the + # lazy one further requires NNUE_PROBE_WEIGHTS to do anything. + if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN" + fi + if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN" + fi + cat > "$NNUE_WRAPPER" <<WRAPPER +#!/usr/bin/env bash +exec "$CANDIDATE" $NNUE_FLAGS "\$@" +WRAPPER + chmod +x "$NNUE_WRAPPER" + CANDIDATE="$NNUE_WRAPPER" +fi + +# NNUE_CANDIDATE_NAME: PGN White/Black identity recorded for the +# candidate engine, forwarded to match_play.py's --candidate-name +# (2026-10-16, see that flag's own comment). Defaults to a name that +# flags "don't trust this as training data" whenever NNUE is in play, +# since self_play_games.pgn is reused later to train the NNUE net +# itself and a differently-calibrated evaluator's scores shouldn't be +# mixed in as labels without this being mechanically filterable. +# Override explicitly if a given NNUE config's scores are actually +# meant to be trusted. +NAME_ARGS="" +if [ -n "${NNUE_WEIGHTS:-}" ]; then + NNUE_CANDIDATE_NAME="${NNUE_CANDIDATE_NAME:-candidate_nnue}" + NAME_ARGS="--candidate-name $NNUE_CANDIDATE_NAME" +fi + +python3 ./match_play.py \ + ${HEAD} \ + ${CANDIDATE} \ + --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \ + --games ${GAMES} \ + --workers 18 \ + --post \ + ${TC_ARGS} \ + ${SPRT_ARGS} \ + ${NAME_ARGS} \ + --scratch /usr/local/tmp/typhoon_match_${LABEL} \ + --log ${LABEL}.log \ + "$@" diff --git a/src/eval_tune/relabel_selfplay_pgn.py b/src/eval_tune/relabel_selfplay_pgn.py new file mode 100644 index 0000000..be14e22 --- /dev/null +++ b/src/eval_tune/relabel_selfplay_pgn.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +One-off (but reusable) maintenance tool: retroactively rename the +White/Black PGN header value for games matching a given Event + Date +list, in place, without touching anything else in the file (moves, +comments, other headers byte-for-byte unchanged). + +Why this exists, 2026-10-16: self_play_games.pgn is reused as NNUE +training data (see docs/NNUE.md). match_play.py recorded every +candidate engine as literally "candidate" regardless of what that +candidate actually was, so the 2026-10-15/2026-10-16 batches (an NNUE +candidate vs the HCE baseline, scores on the NNUE side compressed/ +uncalibrated -- see the engine-vs-engine eval discussion that same +day) are indistinguishable from any other "candidate" batch by header +alone. match_play.py now takes --candidate-name/--baseline-name to +avoid this going forward; this script fixes the games written before +that existed. + +Deliberately stream-edits the file line by line rather than round- +tripping through python-chess's PGN writer -- re-serializing ~190k +games risks reformatting move/comment text (whitespace, line wrapping) +that downstream positional parsers (extract_positions.py) may depend +on being byte-identical to how it was originally written. This script +only ever rewrites a `[White "..."]`/`[Black "..."]` header line, and +only when that game's own `[Date ...]`/`[Event ...]` headers already +matched the target. + +Usage: + python3 relabel_selfplay_pgn.py <pgn-file> \\ + --dates 2026-10-15 2026-10-16 \\ + --event "typhoon self-play match" \\ + --old-name candidate --new-name candidate_nnue_loose \\ + [--dry-run] + +Writes to a sibling temp file and atomically replaces the original +only after a full pass succeeds -- a failure partway through never +leaves the original file half-rewritten. +""" +import argparse +import os +import re +import sys + + +HEADER_RE = re.compile(r'^\[(\w+)\s+"((?:[^"\\]|\\.)*)"\]\s*$') + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("pgn_file") + ap.add_argument("--dates", nargs="+", required=True, + help="Date header values (as PGN prints them, e.g. " + "2026-10-15) whose games are in scope") + ap.add_argument("--event", required=True, + help="Event header value a game must also match") + ap.add_argument("--old-name", required=True, + help="White/Black header value to rename") + ap.add_argument("--new-name", required=True, + help="replacement header value") + ap.add_argument("--dry-run", action="store_true", + help="report what would change, write nothing") + args = ap.parse_args() + + target_dates = set(args.dates) + tmp_path = args.pgn_file + ".relabel_tmp" + + cur_date = None + cur_event = None + in_scope = False # true once we know this game's headers match + games_seen = 0 + games_touched = 0 + renames = 0 + + with open(args.pgn_file, "r", encoding="utf-8", errors="surrogateescape") as fin, \ + open(tmp_path, "w", encoding="utf-8", errors="surrogateescape") as fout: + for line in fin: + m = HEADER_RE.match(line) + if m: + tag, value = m.group(1), m.group(2) + if tag == "Event": + # New header block -> new game starting. + games_seen += 1 + cur_event = value + cur_date = None + in_scope = False + elif tag == "Date": + cur_date = value + in_scope = (cur_event == args.event and cur_date in target_dates) + elif tag in ("White", "Black") and in_scope and value == args.old_name: + renames += 1 + games_touched += 1 if tag == "White" else 0 # count once/game is enough + if not args.dry_run: + line = f'[{tag} "{args.new_name}"]\n' + fout.write(line) + + if args.dry_run: + os.remove(tmp_path) + print(f"[dry run] {games_seen} games scanned, {renames} header " + f"values would be renamed ({args.old_name!r} -> " + f"{args.new_name!r}) for Event={args.event!r} Date in " + f"{sorted(target_dates)}") + return + + os.replace(tmp_path, args.pgn_file) + print(f"{games_seen} games scanned, {renames} header values renamed " + f"({args.old_name!r} -> {args.new_name!r}) for Event={args.event!r} " + f"Date in {sorted(target_dates)}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/eval_tune/stockfish_training_corpus.sh b/src/eval_tune/stockfish_training_corpus.sh new file mode 100755 index 0000000..75ce029 --- /dev/null +++ b/src/eval_tune/stockfish_training_corpus.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +python3 ./match_play.py \ + ../stockfish_xboard ../stockfish_xboard \ + --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \ + --pgn-out ../../pgn/stockfish_self_play_training_data.pgn \ + --games 100000 \ + --workers 100 \ + --sd 10 \ + --post \ + --no-color-swap \ + --scratch /usr/local/tmp/typhoon_match_sf_corpus \ + --log /tmp/typhoon/sf_training_self_play.log diff --git a/src/eval_tune/test_vs_head.sh b/src/eval_tune/test_vs_head.sh index 71b2282..fa44d44 100755 --- a/src/eval_tune/test_vs_head.sh +++ b/src/eval_tune/test_vs_head.sh @@ -75,12 +75,41 @@ # engine_args to both sides): # # NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh +# +# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by +# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/ +# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to +# also be set (the probe gate only does anything once NNUE eval itself +# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the +# same wrapper, enabling the lazy-eval accumulator probe gate at +# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE +# automatically, same convention as --nnueweights): +# +# NNUE_WEIGHTS=/path/to/net.nnue \ +# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \ +# test_vs_head.sh set -e -HEAD="${1:-../../head_reference/typhoon}" -CANDIDATE_ARG="${2:-}" -LABEL="${3:-match}" +# Only consume LEADING args that don't look like a flag as the three +# positionals (head/candidate/label) -- 2026-10-19 fix: a bare +# ./test_vs_head.sh --sd 10 used to land "--sd" in HEAD and "10" in +# CANDIDATE_ARG (since the old code unconditionally took "$1"/"$2"/"$3" +# regardless of content), the exact same footgun class the comment +# below already describes once biting the --shift-then-test-"$2" +# version of this script. A flag-looking first arg now means "no +# positionals given at all, everything is pass-through" instead. +POSITIONALS=() +while [ $# -gt 0 ] && [ "${#POSITIONALS[@]}" -lt 3 ]; do + case "$1" in + -*) break ;; + *) POSITIONALS+=("$1"); shift ;; + esac +done + +HEAD="${POSITIONALS[0]:-../../head_reference/typhoon}" +CANDIDATE_ARG="${POSITIONALS[1]:-}" +LABEL="${POSITIONALS[2]:-match}" # Note the `-` (not `:-`) for the three env overrides below: an # explicitly EMPTY value must be honored, not replaced by the default. # SPRT_ARGS="" is the documented way to run without a hypothesis test @@ -90,18 +119,17 @@ TC_ARGS="${TC_ARGS---st 4}" SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}" GAMES="${GAMES-10000}" -# Drop the (up to) three positional args so "$@" holds only extra -# pass-through flags for match_play.py. This MUST come after the three -# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above: -# an earlier version of this shifted first and then tested "$2" to -# decide whether to rebuild, which (a) made every no-extra-args run -# take the rebuild branch and silently clobber ../typhoon with a -# different make profile, and (b) made the first pass-through FLAG VALUE -# land in "$2" and get used as the candidate BINARY PATH -- producing a -# baffling "PermissionError: ... Permission denied: <scratch dir>" from -# Popen trying to execute a directory. -for _ in 1 2 3; do - if [ $# -gt 0 ]; then shift; fi +# If the caller already passed --sd or --st through as an extra +# argument (e.g. ./test_vs_head.sh --sd 10), don't ALSO inject the +# baked-in TC_ARGS ("--st 4" by default) -- match_play.py's --sd/--st +# are mutually exclusive, so the two would collide +# ("error: argument --st: not allowed with argument --sd") even though +# the caller only asked for one of them. Pass-through wins; TC_ARGS is +# only the fallback when the caller didn't specify either. +for arg in "$@"; do + case "$arg" in + --sd|--st) TC_ARGS="" ;; + esac done if [ -n "$CANDIDATE_ARG" ]; then @@ -117,14 +145,47 @@ fi if [ -n "${NNUE_WEIGHTS:-}" ]; then NNUE_WRAPPER=/tmp/typhoon/typhoon_match_play_candidate_nnue + NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\"" + if [ -n "${NNUE_PROBE_WEIGHTS:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\"" + fi + # NNUE_LAZY_MARGIN / NNUE_SUPERLAZY_MARGIN: 2026-10-16 looser- + # margin experiment (see docs/NNUE.md and command.c's "nnue + # lazymargin"/"nnue superlazyon"). NNUE_LAZY_MARGIN overrides every + # material bucket's lazy-probe margin with one flat cp value + # instead of the calibrated p98 table; NNUE_SUPERLAZY_MARGIN turns + # on the even-cheaper material-only pre-filter (off by default) + # with the given flat cp margin. Both require NNUE_WEIGHTS; the + # lazy one further requires NNUE_PROBE_WEIGHTS to do anything. + if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN" + fi + if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN" + fi cat > "$NNUE_WRAPPER" <<WRAPPER #!/usr/bin/env bash -exec "$CANDIDATE" --nnueweights "$NNUE_WEIGHTS" "\$@" +exec "$CANDIDATE" $NNUE_FLAGS "\$@" WRAPPER chmod +x "$NNUE_WRAPPER" CANDIDATE="$NNUE_WRAPPER" fi +# NNUE_CANDIDATE_NAME: PGN White/Black identity recorded for the +# candidate engine, forwarded to match_play.py's --candidate-name +# (2026-10-16, see that flag's own comment). Defaults to a name that +# flags "don't trust this as training data" whenever NNUE is in play, +# since self_play_games.pgn is reused later to train the NNUE net +# itself and a differently-calibrated evaluator's scores shouldn't be +# mixed in as labels without this being mechanically filterable. +# Override explicitly if a given NNUE config's scores are actually +# meant to be trusted. +NAME_ARGS="" +if [ -n "${NNUE_WEIGHTS:-}" ]; then + NNUE_CANDIDATE_NAME="${NNUE_CANDIDATE_NAME:-candidate_nnue}" + NAME_ARGS="--candidate-name $NNUE_CANDIDATE_NAME" +fi + python3 ./match_play.py \ ${HEAD} \ ${CANDIDATE} \ @@ -134,6 +195,7 @@ python3 ./match_play.py \ --post \ ${TC_ARGS} \ ${SPRT_ARGS} \ + ${NAME_ARGS} \ --scratch /usr/local/tmp/typhoon_match_${LABEL} \ --log ${LABEL}.log \ "$@" diff --git a/src/eval_tune/typhoon_hce_training_corpus.sh b/src/eval_tune/typhoon_hce_training_corpus.sh new file mode 100755 index 0000000..1fc16d8 --- /dev/null +++ b/src/eval_tune/typhoon_hce_training_corpus.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# +# Gated self-play match against a baseline binary. Historically this +# always compared the live working-tree build against head_reference/ +# -- generalized (2026-09-14) to take both binary paths as args so two +# arbitrary builds (e.g. a file-storm-only worktree build vs. the +# current working tree, to isolate one change at a time when several +# have landed together -- see king_safety_data/HANDOFF.md's session +# notes on not measuring two eval changes in one match) can be compared +# without hand-rolling a match_play.py invocation each time. +# +# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label] +# +# head_binary Baseline engine. Defaults to a fresh copy of the +# SAME just-built working-tree binary the candidate +# uses (/tmp/typhoon/typhoon_match_play_head) -- +# this is self-play training-data generation, not a +# "better than last checkpoint" gate, so baseline +# and candidate must share every runtime fix/option +# (e.g. the PostThrottle >= fix); a stale +# head_reference/typhoon silently lacks those and +# produces training data with gaps that look like +# search bugs but are actually just an old binary. +# Pass an explicit path (already built) -- together +# with candidate_binary -- to skip the rebuild/copy +# step entirely, e.g. when comparing two prebuilt +# worktree binaries. +# candidate_binary Candidate engine. Defaults to a fresh copy of the +# live working-tree build (../typhoon) into +# /tmp/typhoon/typhoon_match_play_candidate. Must be +# given together with head_binary -- either both +# positionals are prebuilt paths, or neither is and +# both get rebuilt copies. +# label Distinguishes scratch dir/log/pgn-out so multiple +# matches can run concurrently without colliding. +# Defaults to "match". +# +# ...and any further arguments are passed straight through to +# match_play.py, so a one-off flag no longer needs this file edited. +# +# Three env vars override the baked-in defaults (2026-10-03), so the +# common knobs are reachable without touching the invocation below: +# +# TC_ARGS time control. Default: "--st 4" +# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5" +# GAMES game cap. Default: 10000 +# +# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5" +# cannot resolve the effect sizes this project's LMR work actually +# produces. At the observed draw rate the standard error in Elo is about +# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000 +# games (~53h here) -- and every LMR match run to date has instead +# wandered around llr 0 for 2,000+ games and been abandoned, which is +# exactly what a true effect sitting BETWEEN the two hypotheses looks +# like. For a change you would only ship at >= 15 Elo, use +# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours +# instead of never. +# +# Examples: +# test_vs_head.sh # old default behavior +# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight +# +# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo: +# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh +# +# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the +# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in +# # THIS engine, which bounds what any depth-buying search change +# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity +# # rather than testing a hypothesis, and ELO=+/- prints either way. +# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo +# # of precision already separates "a ply is worth 25" from "worth 65". +# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \ +# test_vs_head.sh ../../head_reference/typhoon \ +# ../../head_reference/typhoon timeodds +# +# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/ +# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped +# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS" +# ahead of whatever args match_play.py's Engine class passes it -- +# that's the only change needed, since Engine spawns the given path +# directly via Popen (no shell), so a shebang script in its place works +# unmodified. HEAD is untouched, so it keeps running the classic HCE -- +# this is how you get an NNUE-candidate-vs-HCE-reference match without +# patching match_play.py's Engine (which currently passes identical +# engine_args to both sides): +# +# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh +# +# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by +# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/ +# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to +# also be set (the probe gate only does anything once NNUE eval itself +# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the +# same wrapper, enabling the lazy-eval accumulator probe gate at +# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE +# automatically, same convention as --nnueweights): +# +# NNUE_WEIGHTS=/path/to/net.nnue \ +# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \ +# test_vs_head.sh + +set -e + +HEAD_ARG="${1:-}" +CANDIDATE_ARG="${2:-}" +LABEL="${3:-match}" +# Note the `-` (not `:-`) for the three env overrides below: an +# explicitly EMPTY value must be honored, not replaced by the default. +# SPRT_ARGS="" is the documented way to run without a hypothesis test +# (e.g. the time-odds calibration above), and `:-` would silently put +# --sprt back. +#TC_ARGS="${TC_ARGS---st 4}" +#SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}" +GAMES="${GAMES-10000}" + +# Drop the (up to) three positional args so "$@" holds only extra +# pass-through flags for match_play.py. This MUST come after the three +# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above: +# an earlier version of this shifted first and then tested "$2" to +# decide whether to rebuild, which (a) made every no-extra-args run +# take the rebuild branch and silently clobber ../typhoon with a +# different make profile, and (b) made the first pass-through FLAG VALUE +# land in "$2" and get used as the candidate BINARY PATH -- producing a +# baffling "PermissionError: ... Permission denied: <scratch dir>" from +# Popen trying to execute a directory. +for _ in 1 2 3; do + if [ $# -gt 0 ]; then shift; fi +done + +if [ -n "$HEAD_ARG" ] && [ -n "$CANDIDATE_ARG" ]; then + HEAD="$HEAD_ARG" + CANDIDATE="$CANDIDATE_ARG" +else + cd .. + gmake clean + gmake release MP=1 PERF_COUNTERS=1 + cd eval_tune + # Both sides get a fresh copy of the SAME just-built ../typhoon -- + # unlike head_reference/typhoon (a stale checked-in binary), this + # guarantees baseline and candidate share every runtime fix/option + # (e.g. the PostThrottle >= fix) so a missing comment in training + # data can't be blamed on baseline running old code. Pass both + # HEAD_ARG and CANDIDATE_ARG explicitly to compare two prebuilt + # binaries instead. + HEAD="${HEAD_ARG:-/tmp/typhoon/typhoon_match_play_head}" + CANDIDATE="${CANDIDATE_ARG:-/tmp/typhoon/typhoon_match_play_candidate}" + cp -f ../typhoon ${HEAD} + cp -f ../typhoon ${CANDIDATE} +fi + +# --no-color-swap: head and candidate are copies of the SAME binary +# here (this is corpus generation, not an Elo gate), so match_play.py's +# default color-swap-per-opening would just play every opening TWICE +# and produce a byte-for-byte duplicate game the second time (confirmed +# 2026-10-16 against nnue_self_play_training_data.pgn) -- pure wasted +# CPU for a training corpus that wants variety, not duplication. See +# --no-color-swap's own help text in match_play.py for the full +# reasoning. +python3 ./match_play.py \ + ${HEAD} \ + ${CANDIDATE} \ + --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \ + --pgn-out ../../pgn/nnue_self_play_training_data.pgn \ + --games ${GAMES} \ + --workers 26 \ + --sd 10 \ + --post \ + --no-color-swap \ + ${NAME_ARGS} \ + --scratch /usr/local/tmp/typhoon_match_${LABEL} \ + --log /tmp/typhoon/nnue_training_self_play.log \ + "$@" diff --git a/src/eval_tune/typhoon_nnue_training_corpus.sh b/src/eval_tune/typhoon_nnue_training_corpus.sh new file mode 100755 index 0000000..b1a00d4 --- /dev/null +++ b/src/eval_tune/typhoon_nnue_training_corpus.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# +# Gated self-play match against a baseline binary. Historically this +# always compared the live working-tree build against head_reference/ +# -- generalized (2026-09-14) to take both binary paths as args so two +# arbitrary builds (e.g. a file-storm-only worktree build vs. the +# current working tree, to isolate one change at a time when several +# have landed together -- see king_safety_data/HANDOFF.md's session +# notes on not measuring two eval changes in one match) can be compared +# without hand-rolling a match_play.py invocation each time. +# +# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label] +# +# head_binary Baseline engine. Defaults to a fresh copy of the +# SAME just-built working-tree binary the candidate +# uses (/tmp/typhoon/typhoon_match_play_head) -- +# this is self-play training-data generation, not a +# "better than last checkpoint" gate, so baseline +# and candidate must share every runtime fix/option +# (e.g. the PostThrottle >= fix); a stale +# head_reference/typhoon silently lacks those and +# produces training data with gaps that look like +# search bugs but are actually just an old binary. +# Pass an explicit path (already built) -- together +# with candidate_binary -- to skip the rebuild/copy +# step entirely, e.g. when comparing two prebuilt +# worktree binaries. +# candidate_binary Candidate engine. Defaults to a fresh copy of the +# live working-tree build (../typhoon) into +# /tmp/typhoon/typhoon_match_play_candidate. Must be +# given together with head_binary -- either both +# positionals are prebuilt paths, or neither is and +# both get rebuilt copies. +# label Distinguishes scratch dir/log/pgn-out so multiple +# matches can run concurrently without colliding. +# Defaults to "match". +# +# ...and any further arguments are passed straight through to +# match_play.py, so a one-off flag no longer needs this file edited. +# +# Three env vars override the baked-in defaults (2026-10-03), so the +# common knobs are reachable without touching the invocation below: +# +# TC_ARGS time control. Default: "--st 4" +# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5" +# GAMES game cap. Default: 10000 +# +# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5" +# cannot resolve the effect sizes this project's LMR work actually +# produces. At the observed draw rate the standard error in Elo is about +# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000 +# games (~53h here) -- and every LMR match run to date has instead +# wandered around llr 0 for 2,000+ games and been abandoned, which is +# exactly what a true effect sitting BETWEEN the two hypotheses looks +# like. For a change you would only ship at >= 15 Elo, use +# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours +# instead of never. +# +# Examples: +# test_vs_head.sh # old default behavior +# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight +# +# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo: +# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh +# +# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the +# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in +# # THIS engine, which bounds what any depth-buying search change +# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity +# # rather than testing a hypothesis, and ELO=+/- prints either way. +# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo +# # of precision already separates "a ply is worth 25" from "worth 65". +# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \ +# test_vs_head.sh ../../head_reference/typhoon \ +# ../../head_reference/typhoon timeodds +# +# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/ +# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped +# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS" +# ahead of whatever args match_play.py's Engine class passes it -- +# that's the only change needed, since Engine spawns the given path +# directly via Popen (no shell), so a shebang script in its place works +# unmodified. HEAD is untouched, so it keeps running the classic HCE -- +# this is how you get an NNUE-candidate-vs-HCE-reference match without +# patching match_play.py's Engine (which currently passes identical +# engine_args to both sides): +# +# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh +# +# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by +# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/ +# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to +# also be set (the probe gate only does anything once NNUE eval itself +# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the +# same wrapper, enabling the lazy-eval accumulator probe gate at +# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE +# automatically, same convention as --nnueweights): +# +# NNUE_WEIGHTS=/path/to/net.nnue \ +# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \ +# test_vs_head.sh + +set -e + +HEAD_ARG="${1:-}" +CANDIDATE_ARG="${2:-}" +LABEL="${3:-match}" +# Note the `-` (not `:-`) for the three env overrides below: an +# explicitly EMPTY value must be honored, not replaced by the default. +# SPRT_ARGS="" is the documented way to run without a hypothesis test +# (e.g. the time-odds calibration above), and `:-` would silently put +# --sprt back. +#TC_ARGS="${TC_ARGS---st 4}" +#SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}" +GAMES="${GAMES-10000}" + +# Drop the (up to) three positional args so "$@" holds only extra +# pass-through flags for match_play.py. This MUST come after the three +# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above: +# an earlier version of this shifted first and then tested "$2" to +# decide whether to rebuild, which (a) made every no-extra-args run +# take the rebuild branch and silently clobber ../typhoon with a +# different make profile, and (b) made the first pass-through FLAG VALUE +# land in "$2" and get used as the candidate BINARY PATH -- producing a +# baffling "PermissionError: ... Permission denied: <scratch dir>" from +# Popen trying to execute a directory. +for _ in 1 2 3; do + if [ $# -gt 0 ]; then shift; fi +done + +if [ -n "$HEAD_ARG" ] && [ -n "$CANDIDATE_ARG" ]; then + HEAD="$HEAD_ARG" + CANDIDATE="$CANDIDATE_ARG" +else + cd .. + gmake clean + gmake release MP=1 PERF_COUNTERS=1 + cd eval_tune + # Both sides get a fresh copy of the SAME just-built ../typhoon -- + # unlike head_reference/typhoon (a stale checked-in binary), this + # guarantees baseline and candidate share every runtime fix/option + # (e.g. the PostThrottle >= fix) so a missing comment in training + # data can't be blamed on baseline running old code. Pass both + # HEAD_ARG and CANDIDATE_ARG explicitly to compare two prebuilt + # binaries instead. + HEAD="${HEAD_ARG:-/tmp/typhoon/typhoon_match_play_nnue1}" + CANDIDATE="${CANDIDATE_ARG:-/tmp/typhoon/typhoon_match_play_nnue2}" + cp -f ../typhoon ${HEAD} + cp -f ../typhoon ${CANDIDATE} +fi + +# --no-color-swap: head and candidate are copies of the SAME binary +# here (this is corpus generation, not an Elo gate), so match_play.py's +# default color-swap-per-opening would just play every opening TWICE +# and produce a byte-for-byte duplicate game the second time (confirmed +# 2026-10-16 against nnue_self_play_training_data.pgn) -- pure wasted +# CPU for a training corpus that wants variety, not duplication. See +# --no-color-swap's own help text in match_play.py for the full +# reasoning. +NNUE_WEIGHTS="${NNUE_WEIGHTS:-$HOME/typhoon/annotate_pgns/runs/sf98k/nnue_ckpt/nnue_weights.nnue}" +if [ ! -f "$NNUE_WEIGHTS" ]; then + echo "typhoon_nnue: NNUE_WEIGHTS file not found: $NNUE_WEIGHTS" >&2 + exit 1 +fi + +NNUE_PROBE_WEIGHTS="${NNUE_PROBE_WEIGHTS-}" + +# Both HEAD and CANDIDATE are the same just-built binary (this is NNUE +# self-play training-data generation, not an HCE-vs-NNUE Elo gate), so +# the NNUE flags get wrapped into exec shims for BOTH sides -- a bare +# string-concatenation of path+flags doesn't work here: match_play.py's +# head_engine/candidate_engine are resolved via Path(...).resolve() as a +# single filesystem path, so passing "$HEAD $ARGS" as one argument would +# just produce a nonexistent path containing literal spaces. See +# test_vs_head.sh's NNUE_WRAPPER for the same pattern, used there to wrap +# only the candidate (HCE head vs NNUE candidate); here both sides need +# the net since both sides generate training data as NNUE. +NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\"" +if [ -n "$NNUE_PROBE_WEIGHTS" ] && [ -f "$NNUE_PROBE_WEIGHTS" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\"" + if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN" + fi +fi +if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN" +fi + +HEAD_WRAPPER=/tmp/typhoon/typhoon_match_play_nnue1_wrapped +cat > "$HEAD_WRAPPER" <<WRAPPER +#!/usr/bin/env bash +exec "$HEAD" $NNUE_FLAGS "\$@" +WRAPPER +chmod +x "$HEAD_WRAPPER" +HEAD="$HEAD_WRAPPER" + +CANDIDATE_WRAPPER=/tmp/typhoon/typhoon_match_play_nnue2_wrapped +cat > "$CANDIDATE_WRAPPER" <<WRAPPER +#!/usr/bin/env bash +exec "$CANDIDATE" $NNUE_FLAGS "\$@" +WRAPPER +chmod +x "$CANDIDATE_WRAPPER" +CANDIDATE="$CANDIDATE_WRAPPER" + +# See test_vs_head.sh's NNUE_CANDIDATE_NAME comment: flags the PGN +# White/Black identity so NNUE-evaluated self-play games are +# mechanically filterable out of training data that isn't meant to +# mix differently-calibrated evaluator scores. +NAME_ARGS="--candidate-name ${NNUE_CANDIDATE_NAME:-candidate_nnue} --baseline-name ${NNUE_HEAD_NAME:-head_nnue}" + +python3 ./match_play.py \ + "${HEAD}" \ + "${CANDIDATE}" \ + --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \ + --pgn-out ../../pgn/nnue_self_play_training_data.pgn \ + --games ${GAMES} \ + --workers 26 \ + --sd 10 \ + --post \ + --no-color-swap \ + ${NAME_ARGS} \ + --scratch /usr/local/tmp/typhoon_match_${LABEL} \ + --log /tmp/typhoon/nnue_training_self_play.log \ + "$@" @@ -305,6 +305,7 @@ Return value: g_Options.uNumProcessors = 1; g_Options.fStatusLine = TRUE; g_Options.iResignThreshold = 0; + g_Options.dPostThrottleSec = 0.35; g_Options.u64MaxNodeCount = 0ULL; i = 1; @@ -424,6 +425,53 @@ Return value: } if (fConsumedNextArg) i++; } + else if ((!STRCMPI(pszArg, "--nnueprobeweights")) && fHaveVal) + { + // Same reasoning as --nnueweights above -- loads and + // enables the lazy-eval accumulator probe gate at startup + // for scripted match-play, where there's no interactive + // "nnue probeload"/"nnue lazyprobeon" command pair + // available. Does NOT imply --nnueweights -- pass both to + // get the full NNUE+lazy-gate configuration; this alone + // without NNUE eval enabled would load the probe but never + // reach the code path that consults it. + if (!NnueProbeLoadWeights(pszVal)) + { + UtilPanic(INITIALIZATION_FAILURE, + (void *)"Bad NNUE probe weights file", + (void *)&(pszVal), + (void *)"probeload", + NULL, + __FILE__, __LINE__); + } + else + { + g_fUseNnueLazyProbe = TRUE; + Trace("Loaded NNUE probe weights \"%s\", lazy probe gate enabled.\n", + pszVal); + } + if (fConsumedNextArg) i++; + } + else if ((!STRCMPI(pszArg, "--nnuelazymargin")) && fHaveVal) + { + // Scripted-match-play equivalent of "nnue lazymargin <cp>" + // -- flat override for every material bucket's lazy-probe + // margin, see nnueeval.c/eval.c. 2026-10-16 looser-margin + // experiment. + g_iNnueLazyMarginOverride = atoi(pszVal); + Trace("NNUE lazy margin override set to %d cp.\n", + g_iNnueLazyMarginOverride); + if (fConsumedNextArg) i++; + } + else if ((!STRCMPI(pszArg, "--nnuesuperlazymargin")) && fHaveVal) + { + // Scripted equivalent of "nnue superlazyon <cp>". + g_iNnueSuperLazyMargin = (SCORE)atoi(pszVal); + g_fUseNnueSuperLazy = TRUE; + Trace("NNUE super-lazy pre-filter enabled, margin=%d cp.\n", + g_iNnueSuperLazyMargin); + if (fConsumedNextArg) i++; + } else if ((!STRCMPI(pszArg, "--dnafile")) && fHaveVal) { if (!ReadEvalDNA(pszVal)) diff --git a/src/nnueaccum.c b/src/nnueaccum.c index 8f01692..7a3010f 100644 --- a/src/nnueaccum.c +++ b/src/nnueaccum.c @@ -34,16 +34,12 @@ Author: #include "chess.h" -// Placeholder embedding table -- see chess.h's extern declaration and -// its own comment. Zero-initialized (C's static/global default), so -// every accumulator is currently all-zero regardless of position; -// real weight loading (reading annotate_pgns/nnue_train.py's trained -// checkpoint) is separate, not-yet-built follow-up work. This is -// sufficient to prove incremental-vs-recompute-from-scratch -// correctness, which only depends on both paths reading the SAME -// table consistently, not on its values being meaningful yet. -float g_rgfNnueWeight[NNUE_DIMENSIONS][NNUE_ACCUMULATOR_SIZE]; -float g_rgfNnueBias[NNUE_ACCUMULATOR_SIZE]; +// Quantized embedding table -- see chess.h's extern declaration and +// its own comment. Zero-initialized (C's static/global default) until +// NnueLoadWeights reads a real exported checkpoint. +SHORT g_rgfNnueWeight[NNUE_DIMENSIONS][NNUE_ACCUMULATOR_SIZE]; +SHORT g_rgfNnueBias[NNUE_ACCUMULATOR_SIZE]; +float g_fNnueFtScale = 1.0f; void NnueAccumAddFeature(NNUE_ACCUMULATOR *pAccum, ULONG uPerspective, @@ -57,12 +53,12 @@ Routine description: **/ { int idx = NnueFeatureIndex(uPerspective, cKingOwn, cPieceSquare, pPiece); - const float *pRow = g_rgfNnueWeight[idx]; + const SHORT *pRow = g_rgfNnueWeight[idx]; ULONG i; for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) { - pAccum->rgfValue[i] += pRow[i]; + pAccum->rgiValue[i] += pRow[i]; } } @@ -76,18 +72,18 @@ Routine description: Subtracts one feature's weight row from *pAccum (in place). Hot path -- the exact inverse of NnueAccumAddFeature, so that applying - Add then Remove (or Remove then Add) for the same arguments is a - no-op up to float rounding. + Add then Remove (or Remove then Add) for the same arguments is an + exact no-op (integer arithmetic, no float rounding to worry about). **/ { int idx = NnueFeatureIndex(uPerspective, cKingOwn, cPieceSquare, pPiece); - const float *pRow = g_rgfNnueWeight[idx]; + const SHORT *pRow = g_rgfNnueWeight[idx]; ULONG i; for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) { - pAccum->rgfValue[i] -= pRow[i]; + pAccum->rgiValue[i] -= pRow[i]; } } @@ -125,7 +121,7 @@ Parameters: for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) { - pAccum->rgfValue[i] = g_rgfNnueBias[i]; + pAccum->rgiValue[i] = g_rgfNnueBias[i]; } // 0x88 COORs aren't a simple monotonic range (A1=0x70 > H8=0x07), diff --git a/src/nnueeval.c b/src/nnueeval.c index 6ee57d5..7caadef 100644 --- a/src/nnueeval.c +++ b/src/nnueeval.c @@ -28,13 +28,23 @@ Author: **/ #include "chess.h" +#include <float.h> -float g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE]; +// g_rgfNnueWeight/g_rgfNnueBias (SHORT, quantized) and g_fNnueFtScale +// are defined in nnueaccum.c. The FC head below is quantized to CHAR +// (int8) per docs/NNUE.md's "Quantization plan" step 2 -- biases stay +// float32 (tiny, added post-dequant). g_rgfNnueActScale/WeightScale +// are read from the export file's header; defaults of 1.0f here only +// matter before any real weights are loaded (the all-zero placeholder +// table makes every activation 0 regardless of scale). +CHAR g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE]; float g_rgfNnueFc1Bias[NNUE_L2]; -float g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2]; +CHAR g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2]; float g_rgfNnueFc2Bias[NNUE_L3]; -float g_rgfNnueFc3Weight[NNUE_L3]; +CHAR g_rgfNnueFc3Weight[NNUE_L3]; float g_rgfNnueFc3Bias; +float g_rgfNnueActScale[3] = { 1.0f, 1.0f, 1.0f }; +float g_rgfNnueWeightScale[3] = { 1.0f, 1.0f, 1.0f }; FLAG g_fNnueWeightsLoaded = FALSE; FLAG g_fUseNnueEval = FALSE; @@ -76,10 +86,12 @@ Return value: } if ((1 != fread(szMagic, 4, 1, fp)) || - (0 != memcmp(szMagic, "NNU1", 4))) + (0 != memcmp(szMagic, "NNU3", 4))) { Trace("NnueLoadWeights: %s is not a valid NNUE weights file " - "(bad magic)\n", szFilename); + "(bad magic -- expected \"NNU3\"; an older \"NNU1\"/" + "\"NNU2\" file predates the FC head's int8 quantization " + "and is no longer readable)\n", szFilename); fclose(fp); return FALSE; } @@ -104,9 +116,24 @@ Return value: return FALSE; } -#define READ_OR_FAIL(ptr, count) \ + if (1 != fread(&g_fNnueFtScale, sizeof(g_fNnueFtScale), 1, fp)) + { + Trace("NnueLoadWeights: %s truncated (ft_scale)\n", szFilename); + fclose(fp); + return FALSE; + } + if ((1 != fread(g_rgfNnueActScale, sizeof(g_rgfNnueActScale), 1, fp)) || + (1 != fread(g_rgfNnueWeightScale, sizeof(g_rgfNnueWeightScale), 1, fp))) + { + Trace("NnueLoadWeights: %s truncated (act/weight scales)\n", + szFilename); + fclose(fp); + return FALSE; + } + +#define READ_OR_FAIL(ptr, elsize, count) \ do { \ - if ((size_t)(count) != fread((ptr), sizeof(float), (count), fp)) \ + if ((size_t)(count) != fread((ptr), (elsize), (count), fp)) \ { \ Trace("NnueLoadWeights: %s truncated\n", szFilename); \ fclose(fp); \ @@ -114,14 +141,16 @@ Return value: } \ } while (0) - READ_OR_FAIL(g_rgfNnueWeight, (size_t)NNUE_DIMENSIONS * NNUE_ACCUMULATOR_SIZE); - READ_OR_FAIL(g_rgfNnueBias, NNUE_ACCUMULATOR_SIZE); - READ_OR_FAIL(g_rgfNnueFc1Weight, (size_t)NNUE_L2 * 2 * NNUE_ACCUMULATOR_SIZE); - READ_OR_FAIL(g_rgfNnueFc1Bias, NNUE_L2); - READ_OR_FAIL(g_rgfNnueFc2Weight, (size_t)NNUE_L3 * NNUE_L2); - READ_OR_FAIL(g_rgfNnueFc2Bias, NNUE_L3); - READ_OR_FAIL(g_rgfNnueFc3Weight, NNUE_L3); - READ_OR_FAIL(&g_rgfNnueFc3Bias, 1); + READ_OR_FAIL(g_rgfNnueWeight, sizeof(SHORT), + (size_t)NNUE_DIMENSIONS * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueBias, sizeof(SHORT), NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueFc1Weight, sizeof(CHAR), + (size_t)NNUE_L2 * 2 * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueFc1Bias, sizeof(float), NNUE_L2); + READ_OR_FAIL(g_rgfNnueFc2Weight, sizeof(CHAR), (size_t)NNUE_L3 * NNUE_L2); + READ_OR_FAIL(g_rgfNnueFc2Bias, sizeof(float), NNUE_L3); + READ_OR_FAIL(g_rgfNnueFc3Weight, sizeof(CHAR), NNUE_L3); + READ_OR_FAIL(&g_rgfNnueFc3Bias, sizeof(float), 1); #undef READ_OR_FAIL @@ -163,48 +192,367 @@ Return value: POSITION *pos = &ctx->sPosition; ULONG uStm = pos->uToMove; ULONG uOther = FLIP(uStm); - float rgfConcat[2 * NNUE_ACCUMULATOR_SIZE]; + UCHAR rgbAct1[2 * NNUE_ACCUMULATOR_SIZE]; // quantized fc1 input + UCHAR rgbAct2[NNUE_L2]; // quantized fc2 input + UCHAR rgbAct3[NNUE_L3]; // quantized fc3 input float rgfL2[NNUE_L2]; float rgfL3[NNUE_L3]; float fOut; ULONG i, j; + // Stage 0: dequantize the INT accumulator (scaled by + // g_fNnueFtScale, see chess.h's NNUE_ACCUMULATOR comment) back to + // float, apply ReLU, then requantize to UCHAR at AS1 + // (g_rgfNnueActScale[0]) -- this is "fc1's input" in the Python + // reference. Own-perspective-first concatenation matches the + // side-to-move-relative output convention. for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) { - float v = ctx->sNnueAccum[uStm].rgfValue[i]; - rgfConcat[i] = (v > 0.0f) ? v : 0.0f; + float v = (float)ctx->sNnueAccum[uStm].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + rgbAct1[i] = (UCHAR)MAX(0, MIN(255, (int)(v * g_rgfNnueActScale[0] + 0.5f))); } for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) { - float v = ctx->sNnueAccum[uOther].rgfValue[i]; - rgfConcat[NNUE_ACCUMULATOR_SIZE + i] = (v > 0.0f) ? v : 0.0f; + float v = (float)ctx->sNnueAccum[uOther].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + rgbAct1[NNUE_ACCUMULATOR_SIZE + i] = + (UCHAR)MAX(0, MIN(255, (int)(v * g_rgfNnueActScale[0] + 0.5f))); } + // Stage 1: fc1, int8 dot product (UCHAR activation x CHAR weight) + // accumulated in INT (32-bit) -- up to 512 terms, each at most + // 255*127, nowhere near INT overflow. Dequantize by dividing out + // both scales, add the (unquantized) bias, ReLU, requantize at + // AS2 for fc2's input. for (j = 0; j < NNUE_L2; j++) { - float s = g_rgfNnueFc1Bias[j]; + INT iSum = 0; for (i = 0; i < 2 * NNUE_ACCUMULATOR_SIZE; i++) { - s += g_rgfNnueFc1Weight[j][i] * rgfConcat[i]; + iSum += (INT)rgbAct1[i] * (INT)g_rgfNnueFc1Weight[j][i]; } - rgfL2[j] = (s > 0.0f) ? s : 0.0f; + float s = ((float)iSum / (g_rgfNnueActScale[0] * g_rgfNnueWeightScale[0])) + + g_rgfNnueFc1Bias[j]; + s = (s > 0.0f) ? s : 0.0f; + rgfL2[j] = s; + rgbAct2[j] = (UCHAR)MAX(0, MIN(255, (int)(s * g_rgfNnueActScale[1] + 0.5f))); } + // Stage 2: fc2, same pattern. for (j = 0; j < NNUE_L3; j++) { - float s = g_rgfNnueFc2Bias[j]; + INT iSum = 0; for (i = 0; i < NNUE_L2; i++) { - s += g_rgfNnueFc2Weight[j][i] * rgfL2[i]; + iSum += (INT)rgbAct2[i] * (INT)g_rgfNnueFc2Weight[j][i]; } - rgfL3[j] = (s > 0.0f) ? s : 0.0f; + float s = ((float)iSum / (g_rgfNnueActScale[1] * g_rgfNnueWeightScale[1])) + + g_rgfNnueFc2Bias[j]; + s = (s > 0.0f) ? s : 0.0f; + rgfL3[j] = s; + rgbAct3[j] = (UCHAR)MAX(0, MIN(255, (int)(s * g_rgfNnueActScale[2] + 0.5f))); } - fOut = g_rgfNnueFc3Bias; - for (i = 0; i < NNUE_L3; i++) + // Stage 3: fc3, single output, no ReLU (final scalar score). { - fOut += g_rgfNnueFc3Weight[i] * rgfL3[i]; + INT iSum = 0; + for (i = 0; i < NNUE_L3; i++) + { + iSum += (INT)rgbAct3[i] * (INT)g_rgfNnueFc3Weight[i]; + } + fOut = ((float)iSum / (g_rgfNnueActScale[2] * g_rgfNnueWeightScale[2])) + + g_rgfNnueFc3Bias; } return (SCORE)(fOut * 100.0f); } + + +void +NnueDumpActivationStats(SEARCHER_THREAD_CONTEXT *ctx) +/** + +Routine description: + + Diagnostic-only: re-runs NnueForward's same math but reports, per + layer, what fraction of quantized activations are clamped to the + [0,255] UCHAR rail (saturated) vs sitting strictly inside it, plus + the pre-clamp float stats. Built in response to the "loses a whole + queen and the eval barely moves" bug report, 2026-10-17 -- the + working theory is that ordinary middlegame positions saturate most + activations to the same rail regardless of material, so a single + piece's worth of accumulator change never survives the clamp to + reach the dense layers; only an extreme (near-decisive) material + swing moves enough units across the rail to register. This + function exists to confirm or refute that theory, not to fix + anything -- reference/measurement-only, not wired into search. + +Parameters: + + SEARCHER_THREAD_CONTEXT *ctx + +Return value: + + void + +**/ +{ + POSITION *pos = &ctx->sPosition; + ULONG uStm = pos->uToMove; + ULONG uOther = FLIP(uStm); + UCHAR rgbAct1[2 * NNUE_ACCUMULATOR_SIZE]; + UCHAR rgbAct2[NNUE_L2]; + UCHAR rgbAct3[NNUE_L3]; + ULONG i, j; + ULONG uClampLow, uClampHigh; + float fMin, fMax, fSum; + + // Stage 0 (fc1 input) -- same math as NnueForward, but track the + // pre-clamp float range and how many of the 512 entries hit each + // rail. + uClampLow = uClampHigh = 0; + fMin = FLT_MAX; + fMax = -FLT_MAX; + fSum = 0.0f; + for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) + { + float v = (float)ctx->sNnueAccum[uStm].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + float q = v * g_rgfNnueActScale[0] + 0.5f; + if (q <= 0.0f) uClampLow++; + if (q >= 255.0f) uClampHigh++; + if (q < fMin) fMin = q; + if (q > fMax) fMax = q; + fSum += q; + rgbAct1[i] = (UCHAR)MAX(0, MIN(255, (int)q)); + } + for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) + { + float v = (float)ctx->sNnueAccum[uOther].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + float q = v * g_rgfNnueActScale[0] + 0.5f; + if (q <= 0.0f) uClampLow++; + if (q >= 255.0f) uClampHigh++; + if (q < fMin) fMin = q; + if (q > fMax) fMax = q; + fSum += q; + rgbAct1[NNUE_ACCUMULATOR_SIZE + i] = (UCHAR)MAX(0, MIN(255, (int)q)); + } + Trace("fc1 input (%u units): %u at 0, %u at 255, range [%.1f, %.1f], mean %.1f\n", + 2 * NNUE_ACCUMULATOR_SIZE, uClampLow, uClampHigh, fMin, fMax, + fSum / (float)(2 * NNUE_ACCUMULATOR_SIZE)); + + // Stage 1 (fc2 input) + uClampLow = uClampHigh = 0; + fMin = FLT_MAX; + fMax = -FLT_MAX; + fSum = 0.0f; + for (j = 0; j < NNUE_L2; j++) + { + INT iSum = 0; + for (i = 0; i < 2 * NNUE_ACCUMULATOR_SIZE; i++) + { + iSum += (INT)rgbAct1[i] * (INT)g_rgfNnueFc1Weight[j][i]; + } + float s = ((float)iSum / (g_rgfNnueActScale[0] * g_rgfNnueWeightScale[0])) + + g_rgfNnueFc1Bias[j]; + s = (s > 0.0f) ? s : 0.0f; + float q = s * g_rgfNnueActScale[1] + 0.5f; + if (q <= 0.0f) uClampLow++; + if (q >= 255.0f) uClampHigh++; + if (q < fMin) fMin = q; + if (q > fMax) fMax = q; + fSum += q; + rgbAct2[j] = (UCHAR)MAX(0, MIN(255, (int)q)); + } + Trace("fc2 input (%u units): %u at 0, %u at 255, range [%.1f, %.1f], mean %.1f\n", + (ULONG)NNUE_L2, uClampLow, uClampHigh, fMin, fMax, + fSum / (float)NNUE_L2); + + // Stage 2 (fc3 input) + uClampLow = uClampHigh = 0; + fMin = FLT_MAX; + fMax = -FLT_MAX; + fSum = 0.0f; + for (j = 0; j < NNUE_L3; j++) + { + INT iSum = 0; + for (i = 0; i < NNUE_L2; i++) + { + iSum += (INT)rgbAct2[i] * (INT)g_rgfNnueFc2Weight[j][i]; + } + float s = ((float)iSum / (g_rgfNnueActScale[1] * g_rgfNnueWeightScale[1])) + + g_rgfNnueFc2Bias[j]; + s = (s > 0.0f) ? s : 0.0f; + float q = s * g_rgfNnueActScale[2] + 0.5f; + if (q <= 0.0f) uClampLow++; + if (q >= 255.0f) uClampHigh++; + if (q < fMin) fMin = q; + if (q > fMax) fMax = q; + fSum += q; + rgbAct3[j] = (UCHAR)MAX(0, MIN(255, (int)q)); + } + Trace("fc3 input (%u units): %u at 0, %u at 255, range [%.1f, %.1f], mean %.1f\n", + (ULONG)NNUE_L3, uClampLow, uClampHigh, fMin, fMax, + fSum / (float)NNUE_L3); +} + + +// Calibration-only lazy-eval probe (docs/NNUE.md's "Lazy-evaluation +// tiering plan"). A linear regression fit offline over fc1's own +// input (512-wide: the ReLU'd, concatenated accumulator halves) -- +// NOT a second trained net, just a cheap linear read of a +// representation the real net already produces. Completely separate +// weight file/format from NnueLoadWeights' production .nnue files; +// this is measurement-only infrastructure, consulted only under +// CALIBRATE_MARGIN_SAFETY (see eval.c's Eval()) and never used to +// skip any real computation yet. +float g_rgfNnueProbeMu[2 * NNUE_ACCUMULATOR_SIZE]; +float g_rgfNnueProbeSigma[2 * NNUE_ACCUMULATOR_SIZE]; +float g_rgfNnueProbeWeight[2 * NNUE_ACCUMULATOR_SIZE]; +float g_fNnueProbeBias; +FLAG g_fNnueProbeLoaded = FALSE; +FLAG g_fUseNnueLazyProbe = FALSE; + +// Runtime overrides for experimenting with how loose the NNUE lazy/ +// super-lazy margins can go without rebuilding -- 2026-10-16, see +// docs/NNUE.md "paradox: safety doesn't win games" discussion. +// g_iNnueLazyMarginOverride < 0 means "use eval.c's calibrated +// per-material-bucket table as before"; >= 0 means "use this flat cp +// value for every bucket instead". g_fUseNnueSuperLazy/ +// g_iNnueSuperLazyMargin gate a brand new, even cheaper pre-filter +// (pure material balance, no net at all) ahead of the probe -- this +// was previously measurement-only (see eval.c's CALIBRATE_MARGIN_SAFETY +// block) and is now wired into the real Eval() path for the first time. +INT g_iNnueLazyMarginOverride = -1; +FLAG g_fUseNnueSuperLazy = FALSE; +SCORE g_iNnueSuperLazyMargin = 0; + + +FLAG +NnueProbeLoadWeights(CHAR *szFilename) +/** + +Routine description: + + Loads the lazy-eval probe's weights -- a flat file written by + annotate_pgns/export_nnue_weights.py's sibling fitting script: a + 4-byte magic ("NPRB"), a uint32 dimension (must equal + 2*NNUE_ACCUMULATOR_SIZE), then mu/sigma/weight (each that many + float32s) and a single float32 bias, in that order. + +Parameters: + + CHAR *szFilename + +Return value: + + FLAG : TRUE on success + +**/ +{ + FILE *fp; + CHAR szMagic[4]; + ULONG uDim; + + g_fNnueProbeLoaded = FALSE; + fp = fopen(szFilename, "rb"); + if (NULL == fp) + { + Trace("NnueProbeLoadWeights: could not open %s\n", szFilename); + return FALSE; + } + + if ((1 != fread(szMagic, 4, 1, fp)) || + (0 != memcmp(szMagic, "NPRB", 4))) + { + Trace("NnueProbeLoadWeights: %s is not a valid probe weights " + "file (bad magic)\n", szFilename); + fclose(fp); + return FALSE; + } + if ((1 != fread(&uDim, sizeof(uDim), 1, fp)) || + (uDim != 2 * NNUE_ACCUMULATOR_SIZE)) + { + Trace("NnueProbeLoadWeights: %s dimension %lu doesn't match " + "this build's %lu\n", szFilename, uDim, + (ULONG)(2 * NNUE_ACCUMULATOR_SIZE)); + fclose(fp); + return FALSE; + } + +#define READ_OR_FAIL(ptr, count) \ + do { \ + if ((size_t)(count) != fread((ptr), sizeof(float), (count), fp)) \ + { \ + Trace("NnueProbeLoadWeights: %s truncated\n", szFilename); \ + fclose(fp); \ + return FALSE; \ + } \ + } while (0) + + READ_OR_FAIL(g_rgfNnueProbeMu, 2 * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueProbeSigma, 2 * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueProbeWeight, 2 * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(&g_fNnueProbeBias, 1); + +#undef READ_OR_FAIL + + fclose(fp); + g_fNnueProbeLoaded = TRUE; + Trace("NnueProbeLoadWeights: loaded %s\n", szFilename); + return TRUE; +} + + +SCORE +NnueProbeScore(SEARCHER_THREAD_CONTEXT *ctx) +/** + +Routine description: + + Cheap lazy-eval proxy: dequantizes+ReLUs the accumulator exactly + like NnueForward's stage 0 (same own-perspective-first + concatenation), standardizes by the probe's fitted mu/sigma, then + a single 512-wide dot product -- no fc1/fc2/fc3 at all. One + multiply-add per dimension plus the dequant/ReLU/standardize work, + versus NnueForward's full ~17,440 MACs across three layers -- + see docs/NNUE.md's latency estimate. + +Parameters: + + SEARCHER_THREAD_CONTEXT *ctx + +Return value: + + SCORE : side-to-move-relative, in centipawns, same convention as + NnueForward (so the two are directly comparable) + +**/ +{ + POSITION *pos = &ctx->sPosition; + ULONG uStm = pos->uToMove; + ULONG uOther = FLIP(uStm); + float fSum; + ULONG i; + + fSum = g_fNnueProbeBias; + for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) + { + float v = (float)ctx->sNnueAccum[uStm].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + v = (v - g_rgfNnueProbeMu[i]) / g_rgfNnueProbeSigma[i]; + fSum += v * g_rgfNnueProbeWeight[i]; + } + for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) + { + float v = (float)ctx->sNnueAccum[uOther].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + v = (v - g_rgfNnueProbeMu[NNUE_ACCUMULATOR_SIZE + i]) / + g_rgfNnueProbeSigma[NNUE_ACCUMULATOR_SIZE + i]; + fSum += v * g_rgfNnueProbeWeight[NNUE_ACCUMULATOR_SIZE + i]; + } + + return (SCORE)(fSum * 100.0f); +} diff --git a/src/search.c b/src/search.c index 8404d9d..7d8d686 100644 --- a/src/search.c +++ b/src/search.c @@ -66,6 +66,7 @@ Revision History: **/ #include "chess.h" +#include <math.h> // DumpQSearchFutilityCalibration's sqrt extern ULONG g_uIterateDepth; extern FLAG g_fCanSplit[MAX_PLY_PER_SEARCH]; @@ -713,7 +714,7 @@ Search(IN SEARCHER_THREAD_CONTEXT *ctx, uRawImproveB, uRawEnemyImproveB); if ((uBetaPredictedFHThousandths != 0xFFFF) && - (uBetaPredictedFHThousandths > 95000)) + (uBetaPredictedFHThousandths > 96000)) { INC(ctx->sCounters.tree.u64QuickNullMoveSuccess); @@ -1371,6 +1372,30 @@ Return value: static UINT64 g_uQFutTries[QFUT_GATE_COUNT][QFUT_DIST_BUCKETS][QFUT_POS_BUCKETS][EVAL_POSITIONAL_SOURCE_COUNT]; static UINT64 g_uQFutSurprises[QFUT_GATE_COUNT][QFUT_DIST_BUCKETS][QFUT_POS_BUCKETS][EVAL_POSITIONAL_SOURCE_COUNT]; +// 2026-10-16 diagnostic (see the raw-pre-clamp iFutility capture +// above): is MAX0 clamping iFutility to 0 for most/all of a given +// source's population, making FUTILITY_BASE_MARGIN_BY_SOURCE's exact +// value irrelevant for that source regardless of what it's set to? +static UINT64 g_uQFutRawPreClampCount[EVAL_POSITIONAL_SOURCE_COUNT]; +static UINT64 g_uQFutRawPreClampNonPositive[EVAL_POSITIONAL_SOURCE_COUNT]; +static INT64 g_uQFutRawPreClampSum[EVAL_POSITIONAL_SOURCE_COUNT]; + +// 2026-10-16: mean/stdev/max of the actual surprise MAGNITUDE +// (iScore - iAlpha, when a rejected move's diagnostic re-search +// raised alpha after all) by source -- distinguishes "surprises are +// consistently small (eval noise near the boundary)" from +// "occasional huge swings (real missed tactics, e.g. a mate the +// shallow SEE-based rejection couldn't see)". Not clamped -- a real +// mate score legitimately belongs in this population, not an outlier +// to exclude. +static UINT64 g_uQFutSurpriseMagCount[EVAL_POSITIONAL_SOURCE_COUNT]; +static INT64 g_uQFutSurpriseMagSum[EVAL_POSITIONAL_SOURCE_COUNT]; +static double g_dQFutSurpriseMagSumSq[EVAL_POSITIONAL_SOURCE_COUNT]; +static INT64 g_uQFutSurpriseMagMax[EVAL_POSITIONAL_SOURCE_COUNT]; +// Cheap shape check without a full histogram: how many surprises are +// "clearly a real tactic" (>100cp) vs "plausibly just noise" (<=100cp). +static UINT64 g_uQFutSurpriseMagOver100[EVAL_POSITIONAL_SOURCE_COUNT]; + static ULONG _QFutDistanceBucket(IN SCORE iDistance) /* iDistance: how far short of the relevant threshold this move was @@ -1397,6 +1422,21 @@ _QFutPositionalBucket(IN SCORE iPositional) return(3); } +static CHAR *g_szQFutGateNames[QFUT_GATE_COUNT] = +{ + "generic capture/promo", + "checking capture/promo (VALUE_ROOK)", + "quiet check (VALUE_BISHOP)", +}; +static CHAR *g_szQFutSourceNames[EVAL_POSITIONAL_SOURCE_COUNT] = +{ + "full eval", + "regular lazy", + "super lazy", + "nnue full", + "nnue lazy probe", +}; + static void _QFutDiagnoseReject(IN SEARCHER_THREAD_CONTEXT *ctx, IN ULONG uMoveNum, @@ -1466,24 +1506,54 @@ Return value: g_uQFutTries[uGate][uDistBucket][uPosBucket][uPositionalSource]++; if (iScore > iAlpha) { + SCORE iMagnitude; g_uQFutSurprises[uGate][uDistBucket][uPosBucket][uPositionalSource]++; + + iMagnitude = (SCORE)(iScore - iAlpha); + g_uQFutSurpriseMagCount[uPositionalSource]++; + g_uQFutSurpriseMagSum[uPositionalSource] += (INT64)iMagnitude; + g_dQFutSurpriseMagSumSq[uPositionalSource] += + (double)iMagnitude * (double)iMagnitude; + if ((INT64)iMagnitude > g_uQFutSurpriseMagMax[uPositionalSource]) + { + g_uQFutSurpriseMagMax[uPositionalSource] = (INT64)iMagnitude; + } + if (iMagnitude > 100) + { + g_uQFutSurpriseMagOver100[uPositionalSource]++; + } + + // 2026-10-16: a few concrete real examples, capped, so a + // surprise-rate gap between sources (e.g. nnue full's 7.41% + // vs classic full's ~1.91% in the same gate) can be looked at + // directly instead of only as an aggregate percentage. Only + // samples the two NNUE sources -- classic's own numbers are + // already understood from the original derivation. + if ((uPositionalSource == EVAL_POSITIONAL_SOURCE_NNUE_FULL) || + (uPositionalSource == EVAL_POSITIONAL_SOURCE_NNUE_LAZY)) + { + static ULONG uSampleCount[EVAL_POSITIONAL_SOURCE_COUNT]; + if (uSampleCount[uPositionalSource] < 10) + { + CHAR *pszFen = PositionToFen(&ctx->sPosition); + if (NULL != pszFen) + { + uSampleCount[uPositionalSource]++; + Trace("QFUT_SURPRISE_SAMPLE source=%s gate=%s " + "move=0x%lx iAlpha=%d iScore=%d iDistance=%d " + "fen=%s\n", + g_szQFutSourceNames[uPositionalSource], + g_szQFutGateNames[uGate], + mv.uMove, (int)iAlpha, (int)iScore, (int)iDistance, + pszFen); + SystemFreeMemory(pszFen); + } + } + } } } -static CHAR *g_szQFutGateNames[QFUT_GATE_COUNT] = -{ - "generic capture/promo", - "checking capture/promo (VALUE_ROOK)", - "quiet check (VALUE_BISHOP)", -}; -static CHAR *g_szQFutSourceNames[EVAL_POSITIONAL_SOURCE_COUNT] = -{ - "full eval", - "regular lazy", - "super lazy", -}; - void DumpQSearchFutilityCalibration(void) /** @@ -1612,6 +1682,51 @@ Return value: } } } + + Trace("\nSurprise magnitude (iScore - iAlpha, when a rejected move's " + "diagnostic re-search raised alpha) by source:\n"); + for (s = 0; s < EVAL_POSITIONAL_SOURCE_COUNT; s++) + { + UINT64 u64Count = g_uQFutSurpriseMagCount[s]; + double dMean, dVariance, dStdev; + if (0 == u64Count) + { + Trace(" %s: no surprises\n", g_szQFutSourceNames[s]); + continue; + } + dMean = (double)g_uQFutSurpriseMagSum[s] / (double)u64Count; + dVariance = (g_dQFutSurpriseMagSumSq[s] / (double)u64Count) - + (dMean * dMean); + if (dVariance < 0.0) + { + dVariance = 0.0; + } + dStdev = sqrt(dVariance); + Trace(" %-16s: n=%8" COMPILER_LONGLONG_UNSIGNED_FORMAT + " mean=%7.1f stdev=%7.1f max=%6" COMPILER_LONGLONG_UNSIGNED_FORMAT + " >100cp=%5.2f%%\n", + g_szQFutSourceNames[s], u64Count, dMean, dStdev, + (UINT64)g_uQFutSurpriseMagMax[s], + 100.0 * (double)g_uQFutSurpriseMagOver100[s] / (double)u64Count); + } + + Trace("\nRaw pre-clamp iFutility by source (is MAX0 saturating this " + "source's population, making the margin constant moot?):\n"); + for (s = 0; s < EVAL_POSITIONAL_SOURCE_COUNT; s++) + { + UINT64 u64Count = g_uQFutRawPreClampCount[s]; + if (0 == u64Count) + { + Trace(" %s: no samples\n", g_szQFutSourceNames[s]); + continue; + } + Trace(" %-16s: n=%10" COMPILER_LONGLONG_UNSIGNED_FORMAT + " mean=%7.1f clamped(<=0)=%5.2f%%\n", + g_szQFutSourceNames[s], u64Count, + (double)g_uQFutRawPreClampSum[s] / (double)u64Count, + 100.0 * (double)g_uQFutRawPreClampNonPositive[s] / + (double)u64Count); + } } #endif // CALIBRATE_QSEARCH_FUTILITY @@ -1891,6 +2006,8 @@ static const SCORE FUTILITY_BASE_MARGIN_BY_SOURCE[EVAL_POSITIONAL_SOURCE_COUNT] FUTILITY_BASE_MARGIN_FULL, // EVAL_POSITIONAL_SOURCE_FULL FUTILITY_BASE_MARGIN_LAZY, // EVAL_POSITIONAL_SOURCE_LAZY FUTILITY_BASE_MARGIN_SUPERLAZY, // EVAL_POSITIONAL_SOURCE_SUPERLAZY + FUTILITY_BASE_MARGIN_NNUE_FULL, // EVAL_POSITIONAL_SOURCE_NNUE_FULL + FUTILITY_BASE_MARGIN_NNUE_LAZY, // EVAL_POSITIONAL_SOURCE_NNUE_LAZY }; @@ -2292,6 +2409,23 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx, iFutility = iAlpha - (FUTILITY_BASE_MARGIN_BY_SOURCE[pi->uLastPositionalSource] + rgiPositional[pos->uToMove]) - iEval; +#ifdef CALIBRATE_QSEARCH_FUTILITY + // 2026-10-16 diagnostic: is iFutility actually being clamped + // to 0 by MAX0 below for this source, making the margin + // constant irrelevant regardless of its value? See the + // investigation into NNUE_FULL's margin change having zero + // measured effect. + { + ULONG uSrc = pi->uLastPositionalSource; + ASSERT(uSrc < EVAL_POSITIONAL_SOURCE_COUNT); + g_uQFutRawPreClampCount[uSrc]++; + g_uQFutRawPreClampSum[uSrc] += (INT64)iFutility; + if (iFutility <= 0) + { + g_uQFutRawPreClampNonPositive[uSrc]++; + } + } +#endif iFutility = MAX0(iFutility); #endif } diff --git a/src/split_ep.py b/src/split_ep.py new file mode 100755 index 0000000..557dbec --- /dev/null +++ b/src/split_ep.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Split a script-format .ep_ file (setboard/solution/id/go blocks, as +used by typhoon's `script` command) into two disjoint, shuffled halves +for a train/holdout threshold sweep. + +Usage: split_ep.py <input.ep_> <out_a.ep_> <out_b.ep_> [--seed N] + +Splits by whole position blocks (never mid-position), shuffled with a +fixed seed so the split is reproducible but not biased by the file's +original ordering (ECM.ep_ is not randomly ordered). +""" +import argparse +import random +import sys + + +def read_blocks(path): + """Each block is the lines from one 'setboard' up to and including + the following 'go' line.""" + blocks = [] + current = [] + with open(path) as f: + for line in f: + stripped = line.rstrip("\n") + if stripped.startswith("setboard") and current: + blocks.append(current) + current = [] + current.append(stripped) + if current: + blocks.append(current) + return blocks + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("input") + ap.add_argument("out_a") + ap.add_argument("out_b") + ap.add_argument("--seed", type=int, default=0) + args = ap.parse_args() + + blocks = read_blocks(args.input) + if not blocks: + print(f"ERROR: no position blocks found in {args.input}", file=sys.stderr) + sys.exit(1) + + rng = random.Random(args.seed) + order = list(range(len(blocks))) + rng.shuffle(order) + + mid = len(order) // 2 + half_a = sorted(order[:mid]) + half_b = sorted(order[mid:]) + + with open(args.out_a, "w") as f: + for i in half_a: + f.write("\n".join(blocks[i]) + "\n") + + with open(args.out_b, "w") as f: + for i in half_b: + f.write("\n".join(blocks[i]) + "\n") + + print(f"{len(blocks)} positions in {args.input} -> " + f"{len(half_a)} in {args.out_a}, {len(half_b)} in {args.out_b} " + f"(seed={args.seed})") + + +if __name__ == "__main__": + main() diff --git a/src/testnnue.c b/src/testnnue.c index fdfaf35..4ee516d 100644 --- a/src/testnnue.c +++ b/src/testnnue.c @@ -30,6 +30,12 @@ Abstract: afterward so this test has no side effect on anything that runs after it in the same process. + g_rgfNnueWeight/g_rgfNnueBias are SHORT (quantized) and + NNUE_ACCUMULATOR.rgiValue is INT -- all integer arithmetic, so + incremental-vs-recompute comparison is exact (no float-rounding + tolerance needed, unlike the original float-accumulator version + of this test). + Author: Scott Gasch ([email protected]) 15 Oct 2026 @@ -59,12 +65,12 @@ Routine description: for (j = 0; j < NNUE_ACCUMULATOR_SIZE; j++) { g_rgfNnueWeight[i][j] = - ((float)(i % 97) * 0.1f) + ((float)(j % 13) * 0.01f) + 1.0f; + (SHORT)(((i % 97) * 10) + ((j % 13) * 1) + 100); } } for (j = 0; j < NNUE_ACCUMULATOR_SIZE; j++) { - g_rgfNnueBias[j] = (float)(j % 7) * 0.5f; + g_rgfNnueBias[j] = (SHORT)((j % 7) * 50); } } @@ -95,12 +101,10 @@ Routine description: against ctx->sPosition and compares it element-by-element against the incrementally-maintained ctx->sNnueAccum. Panics (via UtilPanic, same convention as every other self-test) on any - mismatch outside a small float-rounding tolerance -- real - indexing/logic bugs produce large deltas (a whole missing or - extra ~1-100-magnitude weight row), not rounding noise, so the - tolerance only needs to be loose enough to absorb summation-order - differences between the two paths, not loose enough to hide a - real bug. + mismatch. Integer arithmetic throughout (SHORT table, INT + accumulator) means incremental and from-scratch recompute must + match EXACTLY -- no rounding tolerance needed or wanted; any + nonzero delta is a real indexing/logic bug. Parameters: @@ -114,7 +118,6 @@ Return value: **/ { - static const float EPSILON = 0.01f; NNUE_ACCUMULATOR sFresh; ULONG uColor; ULONG i; @@ -124,16 +127,16 @@ Return value: NnueAccumRecompute(&sFresh, &ctx->sPosition, uColor); for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) { - float fDelta = sFresh.rgfValue[i] - ctx->sNnueAccum[uColor].rgfValue[i]; - if ((fDelta > EPSILON) || (fDelta < -EPSILON)) + INT iDelta = sFresh.rgiValue[i] - ctx->sNnueAccum[uColor].rgiValue[i]; + if (iDelta != 0) { Trace("TestNnueAccumulator: MISMATCH at %s, color=%lu, " - "index=%lu: incremental=%f recompute=%f " + "index=%lu: incremental=%ld recompute=%ld " "mv.uMove=0x%lx cFrom=0x%x cTo=0x%x pMoved=%u " "pCaptured=%u pPromoted=%u bvFlags=0x%x FEN=%s\n", szLabel, uColor, i, - ctx->sNnueAccum[uColor].rgfValue[i], - sFresh.rgfValue[i], + (long)ctx->sNnueAccum[uColor].rgiValue[i], + (long)sFresh.rgiValue[i], mv.uMove, mv.cFrom, mv.cTo, mv.pMoved, mv.pCaptured, mv.pPromoted, mv.bvFlags, PositionToFen(&ctx->sPosition)); @@ -1019,7 +1019,6 @@ FinishPVTailFromHash(SEARCHER_THREAD_CONTEXT *ctx, { if (u64VisitedSigs[u] == u64Sig) { - if (uLenRemain > 6) strcat(buf, "<REP>"); goto unmake; } } @@ -1369,7 +1368,9 @@ Return value: // running under xboard, match its expected format. Otherwise // be more helpful / verbose. // - if ((dNow > 0.35) || ((dNow > 0.15) && (g_Options.fVerbosePosting))) + if ((dNow >= g_Options.dPostThrottleSec) || + ((dNow > (0.15 / 0.35) * g_Options.dPostThrottleSec) && + (TRUE == g_Options.fVerbosePosting))) { if (TRUE == g_Options.fRunningUnderXboard) { @@ -143,6 +143,10 @@ USER_VARIABLE g_UserVarList[] = "B", (void *)&(g_Options.fShouldPost), NULL }, + { "PostThrottle", + "D", + (void *)&(g_Options.dPostThrottleSec), + NULL }, { "ResignThreshold", "I", (void *)&(g_Options.iResignThreshold), @@ -442,7 +446,11 @@ Return value: y = atoi(argv[2]); *((int *)g_UserVarList[x].pValue) = y; break; - + + case 'D': + *((double *)g_UserVarList[x].pValue) = atof(argv[2]); + break; + case 'B': if (toupper(*(argv[2])) == 'T') { |
