diff options
| author | Scott Gasch <[email protected]> | 2026-10-20 18:37:20 -0700 |
|---|---|---|
| committer | Scott Gasch <[email protected]> | 2026-10-20 18:37:20 -0700 |
| commit | 9cf14e14e6999420f234cda951894337e034f57c (patch) | |
| tree | d4845d0cf127ef4226a5cd2e60472ee750213614 | |
| parent | c8c6bbc2c1a103e5b07b079a0f8bdf66c7724d80 (diff) | |
NNUE: re-fit raw-to-score calibration, NNU4 weight format, super-lazy default-on, lazy probe measured net-negative
Fits an affine (a + b*raw) correction into every exported .nnue file
(export_nnue_weights.py's CalibrateScoreMapping) so the engine's
consumed centipawn output tracks Stockfish's real score scale rather
than the net's nominal raw WDL-sigmoid output -- training variance
alone shifts this mapping across retrains even with an identical
recipe, so it's re-fit and baked in per export rather than assumed
constant. Weight format bumped to "NNU4" to carry the two new floats;
older NNU1/2/3 files are no longer readable.
Super-lazy (material-only) pre-filter promoted to default-on
(g_fUseNnueSuperLazy = TRUE) with a recalibrated per-material-bucket
margin table (measured against the recalibrated net, 16-shard
cross_sectional_200k.ep_ sample) -- real, measured nps win across all
three curated suites. The separate lazy-probe gate (accumulator-based
early exit) is measured net-negative on real search traffic at this
net's probe-vs-forward latency ratio and stays off by default
(g_fUseNnueLazyProbe = FALSE); code kept live since margin tuning
could still flip this, per docs/NNUE.md.
Also: direct per-call latency instrumentation for NnueProbeScore/
NnueForward (root.c, EVAL_TIME build) to make the lazy-probe
breakeven math measurable directly instead of inferred; QSearch
futility margins given real-data-derived NNUE_FULL/NNUE_LAZY
starting points.
Co-Authored-By: Claude Sonnet 5 <[email protected]>
| -rw-r--r-- | annotate_pgns/build_training_tensors.py | 64 | ||||
| -rw-r--r-- | annotate_pgns/export_nnue_weights.py | 95 | ||||
| -rw-r--r-- | annotate_pgns/nnue_model.py | 84 | ||||
| -rw-r--r-- | annotate_pgns/nnue_train.py | 139 | ||||
| -rw-r--r-- | annotate_pgns/run_nnue_pipeline.py | 57 | ||||
| -rwxr-xr-x | pgn/twic.pl | 6 | ||||
| -rwxr-xr-x | src/chess.h | 25 | ||||
| -rw-r--r-- | src/command.c | 12 | ||||
| -rw-r--r-- | src/docs/NNUE.md | 407 | ||||
| -rw-r--r-- | src/eval.c | 129 | ||||
| -rw-r--r-- | src/nnueeval.c | 68 | ||||
| -rw-r--r-- | src/root.c | 53 |
12 files changed, 980 insertions, 159 deletions
diff --git a/annotate_pgns/build_training_tensors.py b/annotate_pgns/build_training_tensors.py index 0f69d6c..ff9e772 100644 --- a/annotate_pgns/build_training_tensors.py +++ b/annotate_pgns/build_training_tensors.py @@ -51,16 +51,14 @@ import numpy as np from nnue_feature_spec import active_feature_indices, chess -# Standard piece values for the auxiliary material-probe target -# (nnue_model.py's material_probe, added 2026-10-17 after a measured -# failure -- see that module's docstring). Plain material count, -# mover-relative, NOT run through the WDL sigmoid -- this is a -# straight regression target, deliberately non-saturating. Kings -# excluded (always 1-1 on a legal board, carries no signal). -_PIECE_VALUES = { - chess.PAWN: 1, chess.KNIGHT: 3, chess.BISHOP: 3, - chess.ROOK: 5, chess.QUEEN: 9, -} +# Clip bound for the auxiliary score-probe target (nnue_model.py's +# score_probe, added 2026-10-20 -- see that module's docstring). A +# handful of blowout-but-not-forced-mate scores (see the user's own +# +13.67-before-mate example) would otherwise dominate the probe's +# MSE; exact/mate rows are excluded entirely (see SCORE_VALID below), +# not just clipped, since a mate score isn't "the Stockfish score" in +# the pawn sense this probe targets. +SCORE_CLIP_PAWNS = 20.0 def position_target(pos, lam): @@ -78,14 +76,18 @@ def position_target(pos, lam): return lam * p_search + (1.0 - lam) * r_outcome -def material_balance(board): - """Mover-relative material balance in pawns, +ve = side to move - is materially ahead.""" - total = 0 - for piece_type, value in _PIECE_VALUES.items(): - total += value * len(board.pieces(piece_type, chess.WHITE)) - total -= value * len(board.pieces(piece_type, chess.BLACK)) - return total if board.turn == chess.WHITE else -total +def score_probe_target(pos): + """Returns (score, valid) for the auxiliary score-probe target -- + Stockfish's own raw per-position score (pos["score"], already + mover-relative pawns per extract_selfplay_positions.py), clipped + to +-SCORE_CLIP_PAWNS, or (0.0, False) for exact/mate rows (no + "score" field, or not a real Stockfish-judged centipawn-ish value + -- same exclusion reasoning as p_search's, see NNUE.md).""" + if pos.get("exact") or "score" not in pos: + return 0.0, False + s = pos["score"] + s = max(-SCORE_CLIP_PAWNS, min(SCORE_CLIP_PAWNS, s)) + return s, True def iter_positions(path): @@ -124,7 +126,8 @@ def process_chunk(args): black_lens = array.array("i") stm = [] target = [] - material = [] + score_probe_val = [] + score_probe_valid = [] n_total = len(positions) n_skipped = 0 @@ -145,10 +148,12 @@ def process_chunk(args): stm.append(1 if board.turn == chess.WHITE else 0) target.append(t) - material.append(material_balance(board)) + s, valid = score_probe_target(pos) + score_probe_val.append(s) + score_probe_valid.append(valid) return (white_idx, white_lens, black_idx, black_lens, stm, target, - material, n_total, n_skipped) + score_probe_val, score_probe_valid, n_total, n_skipped) def main(): @@ -199,7 +204,8 @@ def main(): black_off = [0] stm = [] # 1 = white to move, 0 = black target = [] - material = [] # auxiliary material-probe target, mover-relative pawns + score_probe_val = [] # auxiliary score-probe target, clipped pawns + score_probe_valid = [] # False for exact/mate rows -- excluded from that loss n_total = 0 n_used = 0 @@ -226,7 +232,9 @@ def main(): stm.append(1 if board.turn == chess.WHITE else 0) target.append(t) - material.append(material_balance(board)) + s, valid = score_probe_target(pos) + score_probe_val.append(s) + score_probe_valid.append(valid) n_used += 1 if n_used % 200000 == 0: @@ -244,7 +252,7 @@ def main(): args.chunk_size)) with Pool(processes=workers) as pool: for (w_idx, w_lens, b_idx, b_lens, chunk_stm, chunk_target, - chunk_material, chunk_total, + chunk_score_val, chunk_score_valid, chunk_total, chunk_skipped) in pool.imap(process_chunk, chunks): n_total += chunk_total n_skipped += chunk_skipped @@ -258,7 +266,8 @@ def main(): stm.extend(chunk_stm) target.extend(chunk_target) - material.extend(chunk_material) + score_probe_val.extend(chunk_score_val) + score_probe_valid.extend(chunk_score_valid) n_used += len(chunk_stm) print(f" ...{n_used} positions processed " @@ -274,7 +283,10 @@ def main(): np.save(outdir / "black_off.npy", np.array(black_off, dtype=np.int64)) np.save(outdir / "stm.npy", np.array(stm, dtype=np.uint8)) np.save(outdir / "target.npy", np.array(target, dtype=np.float32)) - np.save(outdir / "material.npy", np.array(material, dtype=np.float32)) + np.save(outdir / "score_probe_val.npy", + np.array(score_probe_val, dtype=np.float32)) + np.save(outdir / "score_probe_valid.npy", + np.array(score_probe_valid, dtype=np.uint8)) nnz_white = len(white_idx) nnz_black = len(black_idx) diff --git a/annotate_pgns/export_nnue_weights.py b/annotate_pgns/export_nnue_weights.py index 4c8cb99..d26ab93 100644 --- a/annotate_pgns/export_nnue_weights.py +++ b/annotate_pgns/export_nnue_weights.py @@ -4,8 +4,8 @@ the C engine (src/nnueeval.c's NnueLoadWeights) can read directly -- no parsing, just a fixed-order sequence of arrays behind a small header. -File format (all little-endian), version "NNU3": - magic: 4 bytes, ASCII "NNU3" +File format (all little-endian), version "NNU4": + magic: 4 bytes, ASCII "NNU4" dims: 4x uint32 -- NNUE_DIMENSIONS, L1, L2, L3 (sanity-checked against the C side's own compile-time constants on load, so a mismatched build can't silently misread the @@ -24,6 +24,15 @@ File format (all little-endian), version "NNU3": CalibrateActivationScales) with headroom, NOT from a theoretical max -- see docs/NNUE.md's "Quantization plan" / the int8 float-vs-quantized correctness check. + calib: 2x float32 (CALIB_A, CALIB_B) -- affine correction so + the engine's consumed centipawn output tracks + Stockfish's own real score, not the net's nominal raw + WDL-sigmoid scale (which drifts run-to-run even with + an identical K/lambda/loss-exponent recipe -- see + CalibrateScoreMapping and the 2026-10-20 "keeping this + stable across retrains" conversation). NnueForward + applies `(CALIB_A + CALIB_B*fOut) * 100.0f` instead of + a bare `fOut * 100.0f`. w_scale: 3x float32 (WS1, WS2, WS3) -- fixed-point scales for fc1.weight/fc2.weight/fc3.weight (int8), one per layer, each derived from that layer's own max|.| (exact, not @@ -56,7 +65,8 @@ Usage: python3 export_nnue_weights.py \ runs/sd10_twic_2M/nnue_ckpt/ckpt_epoch019.pt \ runs/sd10_twic_2M/nnue_ckpt/weights.nnue \ - --tensor-dir runs/sd10_twic_2M/tensors_train + --tensor-dir runs/sd10_twic_2M/tensors_train \ + --holdout-tensor-dir runs/sd10_twic_2M/tensors_holdout """ import argparse import struct @@ -136,6 +146,74 @@ def CalibrateActivationScales(model, tensor_dir, n_sample=50000, seed=42): return tuple(scales) +def CalibrateScoreMapping(model, holdout_tensor_dir, n_sample=20000, seed=7): + """Fits a simple affine correction (a, b) so that + a + b*raw tracks Stockfish's own real score (pawns, mover- + relative) -- not raw's nominal WDL-sigmoid scale. See the + 2026-10-20 "keeping this stable across retrains" conversation: + a fixed K/lambda/loss-exponent recipe is NOT enough on its own to + guarantee the engine's consumed cp output stays on a stable scale + across generations -- training variance (data, convergence) still + shifts the empirical raw-to-score mapping even with an identical + recipe. Re-fitting (a, b) every generation and baking the + correction into the exported format, applied in NnueForward + itself, decouples engine search-tuning from needing the raw net's + internal scale to stay perfectly constant forever. + + Fit against the HOLDOUT tensor dir specifically (not train) -- + an honest estimate needs positions the checkpoint wasn't fit on. + Uses score_probe_val.npy/score_probe_valid.npy (Stockfish's own + clipped per-position score, and a validity mask excluding exact/ + mate rows -- see build_training_tensors.py), which + build_training_tensors.py already writes for every tensor dir + regardless of whether --score-loss-weight was ever used during + training. Ordinary least squares, free intercept (not forced + through the origin).""" + d = Path(holdout_tensor_dir) + white_idx = np.load(d / "white_idx.npy"); white_off = np.load(d / "white_off.npy") + black_idx = np.load(d / "black_idx.npy"); black_off = np.load(d / "black_off.npy") + stm = np.load(d / "stm.npy") + score_val = np.load(d / "score_probe_val.npy") + score_valid = np.load(d / "score_probe_valid.npy").astype(bool) + + valid_rows = np.flatnonzero(score_valid) + if len(valid_rows) == 0: + sys.exit(f"CalibrateScoreMapping: {d} has no valid score-probe rows " + f"(all exact/mate?) -- can't fit a calibration against it") + + rng = np.random.default_rng(seed) + rows = rng.choice(valid_rows, size=min(n_sample, len(valid_rows)), replace=False) + rows.sort() + + raws = [] + with torch.no_grad(): + for i in range(0, len(rows), 5000): + chunk = rows[i:i + 5000] + w_idx_list, w_off_list, b_idx_list, b_off_list = [], [], [], [] + off_w, off_b = 0, 0 + for r in chunk: + wi = white_idx[white_off[r]:white_off[r + 1]] + bi = black_idx[black_off[r]:black_off[r + 1]] + w_idx_list.append(wi); w_off_list.append(off_w); off_w += len(wi) + b_idx_list.append(bi); b_off_list.append(off_b); off_b += len(bi) + w_idx = torch.from_numpy(np.concatenate(w_idx_list).astype(np.int64)) + w_off = torch.tensor(w_off_list, dtype=torch.int64) + b_idx = torch.from_numpy(np.concatenate(b_idx_list).astype(np.int64)) + b_off = torch.tensor(b_off_list, dtype=torch.int64) + stm_t = torch.from_numpy(stm[chunk].astype(np.int64)) + raws.append(model(w_idx, w_off, b_idx, b_off, stm_t).numpy()) + raw = np.concatenate(raws).astype(np.float64) + score = score_val[rows].astype(np.float64) + + # score ~= a + b*raw, ordinary least squares, free intercept. + b, a = np.polyfit(raw, score, 1) + resid = score - (a + b * raw) + print(f"CalibrateScoreMapping: fit on {len(rows)} holdout positions -- " + f"a={a:.4f}, b={b:.4f}, mean|resid|={np.mean(np.abs(resid)):.3f} pawns, " + f"resid p98={np.percentile(np.abs(resid), 98):.3f} pawns") + return float(a), float(b) + + def QuantizeFt(arr): """Rounds + clamps a float32 array to int16 at FT_SCALE. Returns (int16 ndarray, number of values that clipped).""" @@ -241,6 +319,13 @@ def main(): help="a build_training_tensors.py output dir, used " "to calibrate the FC head's activation scales " "against real positions (e.g. tensors_train)") + ap.add_argument("--holdout-tensor-dir", required=True, + help="a build_training_tensors.py HOLDOUT output dir " + "(not the same as --tensor-dir), used to fit the " + "raw-to-real-Stockfish-score affine correction " + "(a, b) baked into the exported format -- see " + "CalibrateScoreMapping. Must be holdout, not " + "train, for an honest (non-overfit) fit.") args = ap.parse_args() ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=True) @@ -276,6 +361,7 @@ def main(): model.eval() AS1, AS2, AS3 = CalibrateActivationScales(model, args.tensor_dir) + CALIB_A, CALIB_B = CalibrateScoreMapping(model, args.holdout_tensor_dir) ft_weight_f32 = sd["ft.weight"].detach().contiguous().to(torch.float32).numpy() ft_bias_f32 = sd["ft_bias"].detach().contiguous().to(torch.float32).numpy() @@ -302,11 +388,12 @@ def main(): f"(see docs/NNUE.md)") with open(args.outfile, "wb") as f: - f.write(b"NNU3") + f.write(b"NNU4") f.write(struct.pack("<4I", DIMENSIONS, L1, L2, L3)) f.write(struct.pack("<f", FT_SCALE)) f.write(struct.pack("<3f", AS1, AS2, AS3)) f.write(struct.pack("<3f", WS1, WS2, WS3)) + f.write(struct.pack("<2f", CALIB_A, CALIB_B)) f.write(ft_weight_q.tobytes()) f.write(ft_bias_q.tobytes()) f.write(fc1_w_q.tobytes()) diff --git a/annotate_pgns/nnue_model.py b/annotate_pgns/nnue_model.py index c483ce7..47dca76 100644 --- a/annotate_pgns/nnue_model.py +++ b/annotate_pgns/nnue_model.py @@ -19,18 +19,58 @@ prefilter. All irrelevant to proving the training pipeline works. 2026-10-17: added `material_probe`, a small auxiliary linear head on the same concatenated accumulator `fc1` reads, trained jointly with -the main WDL loss (see nnue_train.py's --material-loss-weight) against -each position's raw material balance (mover-relative pawns, no -sigmoid). Added after a real, measured failure: a net trained with -plain MSE (and separately, exponent=2.6) against WDL-sigmoid targets -alone could not tell a lost pawn from a lost queen in crowded midgame -positions (see sanity_check_net.py) -- directionally correct but -magnitude-blind. The probe supplies a non-saturating gradient straight -at material, forcing `ft`'s embedding weights to keep material -linearly recoverable even where the main WDL loss has little pressure -to care. It's training-only scaffolding, like the SF doc's "virtual -features" factorization trick -- dropped before export, zero -inference-time or quantization-format cost. +the main WDL loss against each position's raw material balance +(mover-relative pawns, no sigmoid). Added after a real, measured +failure: a net trained with plain MSE (and separately, exponent=2.6) +against WDL-sigmoid targets alone could not tell a lost pawn from a +lost queen in crowded midgame positions (see sanity_check_net.py) -- +directionally correct but magnitude-blind. + +2026-10-20: replaced by `score_probe` (same mechanism, different +target -- see that section below) and removed. Once `score_probe` +regresses Stockfish's actual raw score directly, `material_probe`'s +target (piece count) is a strict special case of what `score_probe` +already has to recover -- material dominates Stockfish's own score in +almost any position, so a net that fits the real score well has +implicitly recovered material too. Kept as one objective instead of +two overlapping ones; re-add material_probe alongside score_probe only +if a future sanity check shows score_probe alone is still +material-blind in some case it doesn't already cover. + +2026-10-20: tried `score_probe`, same mechanism as material_probe +(small Linear head on the same accumulator, trained jointly, dropped +before export) targeting Stockfish's own raw per-position score +directly via plain MSE -- motivated by a measured failure of a +different kind from material_probe's: in clearly-losing positions, +the main WDL head's score magnitude did not reflect how lost the +position actually was (-2 for "lost" and -3 for "about to be mated", +no meaningful gradation), because the WDL loss only ever supervises +`raw` through `sigmoid(raw/K)` against a win-probability target, which +saturates hard once a position is clearly decided (K=2.0659 on the +2026-10-20 corpus -- sigmoid(8/K)=0.9796 vs sigmoid(13/K)=0.9982, a +target spread of under 0.02 for a 5-pawn difference in the real +score). **Measured, then reverted**: a medium-scale (500k positions, +~15 epochs) controlled comparison (score_probe-weighted run vs. an +otherwise-identical control) showed only a small improvement in how +well `forward()`'s own raw output tracked the real score at the +losing extremes -- compression was still clearly present in both. +Root cause: score_probe's gradient only ever reaches the *shared +accumulator* (`ft`/`ft_p`/`ft_bias`), never `fc1`/`fc2`/`fc3` -- +nothing forces the actually-exported path to widen its own range, it +can only optionally make use of a richer upstream representation if +doing so happens to help the (still-saturating) WDL loss too. + +2026-10-20: replaced score_probe with a direct, unsaturated regression +term on `raw` itself: `MSE(raw, clipped_score)`, added straight +alongside the WDL loss in nnue_train.py, no auxiliary head at all. +This puts gradient directly on `fc1`/`fc2`/`fc3` -- the actual +exported path -- rather than hoping a richer accumulator happens to +propagate through. Not a competing objective with the WDL loss, either: +since `p_search = sigmoid(score/K)`, the WDL loss's own implied ideal +`raw` is already approximately `score` (especially at lambda close to +1.0) -- this term is an unsaturated restatement of the same target, +not a different one, so no separate probe/model-surface-area is +needed to get its benefit. See nnue_train.py's --score-loss-weight. 2026-10-18: added `ft_p`, the actual Stockfish-doc "P" factor (see NNUE.md's "feature factorization" status entry) -- a second, @@ -96,15 +136,9 @@ class NnueNet(nn.Module): self.fc2 = nn.Linear(l2, l3) self.fc3 = nn.Linear(l3, 1) - # Auxiliary, training-only head -- see module docstring's - # 2026-10-17 note. Reads the same concatenated accumulator - # fc1 does; not used at inference/export time. - self.material_probe = nn.Linear(2 * l1, 1) - def accumulator(self, white_idx, white_off, black_idx, black_off, stm): """The ReLU'd, own-perspective-first concatenated accumulator - (shape [B, 2*l1]) -- shared input to both fc1 (the real eval - head) and material_probe (the auxiliary head). ft_p's + (shape [B, 2*l1]) -- input to fc1. ft_p's contribution uses the identical bag offsets as ft -- same positions, same active-feature counts, just a second lookup of each index modulo ps_nb into the smaller, denser table.""" @@ -134,18 +168,6 @@ class NnueNet(nn.Module): x = self.fc3(x) return x.view(-1) - def forward_with_material(self, white_idx, white_off, black_idx, black_off, stm): - """Like forward(), but also returns the auxiliary material - probe's prediction (mover-relative pawns, no sigmoid) off the - same accumulator -- one extra Linear(2*l1, 1), training-only. - Returns (raw_score, material_pred), both shape [B].""" - x = self.accumulator(white_idx, white_off, black_idx, black_off, stm) - material_pred = self.material_probe(x).view(-1) - y = torch.relu(self.fc1(x)) - y = torch.relu(self.fc2(y)) - y = self.fc3(y) - return y.view(-1), material_pred - def win_probability(self, white_idx, white_off, black_idx, black_off, stm, k=DEFAULT_K): raw = self.forward(white_idx, white_off, black_idx, black_off, stm) return torch.sigmoid(raw / k) diff --git a/annotate_pgns/nnue_train.py b/annotate_pgns/nnue_train.py index ea3d0a9..e413aa9 100644 --- a/annotate_pgns/nnue_train.py +++ b/annotate_pgns/nnue_train.py @@ -42,17 +42,22 @@ class CsrTensors: self.black_off = np.load(tensor_dir / "black_off.npy") self.stm = np.load(tensor_dir / "stm.npy") self.target = np.load(tensor_dir / "target.npy") - # material.npy added 2026-10-17 (see nnue_model.py's - # material_probe) -- optional for backward compat with - # tensor dirs built before this field existed. - material_path = tensor_dir / "material.npy" - self.material = (np.load(material_path) if material_path.exists() - else np.zeros_like(self.target)) + # score_probe_val.npy/score_probe_valid.npy added 2026-10-20 + # (see nnue_model.py's score_probe, replacing material_probe) + # -- optional for backward compat with tensor dirs built before + # this field existed. + score_path = tensor_dir / "score_probe_val.npy" + valid_path = tensor_dir / "score_probe_valid.npy" + self.score_probe_val = (np.load(score_path) if score_path.exists() + else np.zeros_like(self.target)) + self.score_probe_valid = (np.load(valid_path) if valid_path.exists() + else np.zeros_like(self.target, dtype=np.uint8)) self.n = len(self.target) assert len(self.white_off) == self.n + 1 assert len(self.black_off) == self.n + 1 assert len(self.stm) == self.n - assert len(self.material) == self.n + assert len(self.score_probe_val) == self.n + assert len(self.score_probe_valid) == self.n def __len__(self): return self.n @@ -93,7 +98,8 @@ def iterate_batches(data: CsrTensors, batch_size, shuffle, rng): b_idx, b_off = _gather_batch(data.black_idx, data.black_off, rows_sorted) stm = data.stm[rows_sorted] target = data.target[rows_sorted] - material = data.material[rows_sorted] + score_probe_val = data.score_probe_val[rows_sorted] + score_probe_valid = data.score_probe_valid[rows_sorted] yield ( torch.from_numpy(w_idx.astype(np.int64, copy=False)), torch.from_numpy(w_off), @@ -101,7 +107,8 @@ def iterate_batches(data: CsrTensors, batch_size, shuffle, rng): torch.from_numpy(b_off), torch.from_numpy(stm.astype(np.int64, copy=False)), torch.from_numpy(target.copy()), - torch.from_numpy(material.astype(np.float32, copy=True)), + torch.from_numpy(score_probe_val.astype(np.float32, copy=True)), + torch.from_numpy(score_probe_valid.astype(np.float32, copy=True)), ) @@ -118,34 +125,63 @@ def powered_loss(pred, target, exponent, reduction="mean"): return err.sum() if reduction == "sum" else err.mean() -def evaluate(model, data, batch_size, k, loss_exponent, device="cpu"): - """Returns (wdl_loss, material_mse) -- kept separate rather than - pre-combined so the log/checkpoint can show whether the material - probe is actually converging, independently of whatever weight +def score_loss_fn(raw, score_val, score_valid, score_loss_type, huber_delta): + if score_loss_type == "huber": + return masked_huber_sum(raw, score_val, score_valid, huber_delta) + return masked_mse_sum(raw, score_val, score_valid) + + +def evaluate(model, data, batch_size, k, loss_exponent, device="cpu", + score_loss_type="mse", huber_delta=1.0): + """Returns (wdl_loss, score_mse) -- kept separate rather than + pre-combined so the log/checkpoint can show whether `raw` is + actually tracking the real score, independently of whatever weight the caller gives it in the training loss.""" model.eval() total_wdl_loss = 0.0 - total_material_loss = 0.0 + total_score_loss = 0.0 total_n = 0 + total_score_valid = 0 rng = np.random.default_rng(0) with torch.no_grad(): - for w_idx, w_off, b_idx, b_off, stm, target, material in iterate_batches( + for w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid in iterate_batches( data, batch_size, shuffle=False, rng=rng): - w_idx, w_off, b_idx, b_off, stm, target, material = _to_device( - (w_idx, w_off, b_idx, b_off, stm, target, material), device) - raw, material_pred = model.forward_with_material(w_idx, w_off, b_idx, b_off, stm) + w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid = _to_device( + (w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid), device) + raw = model.forward(w_idx, w_off, b_idx, b_off, stm) pred = torch.sigmoid(raw / k) wdl_loss = powered_loss(pred, target, loss_exponent, reduction="sum") - material_loss = nn_mse_sum(material_pred, material) + score_loss = score_loss_fn(raw, score_val, score_valid, score_loss_type, huber_delta) total_wdl_loss += wdl_loss.item() - total_material_loss += material_loss.item() + total_score_loss += score_loss.item() total_n += len(target) + total_score_valid += int(score_valid.sum().item()) model.train() - return total_wdl_loss / max(total_n, 1), total_material_loss / max(total_n, 1) + return (total_wdl_loss / max(total_n, 1), + total_score_loss / max(total_score_valid, 1)) + + +def masked_mse_sum(pred, target, valid): + """Sum of squared error over only the rows where valid != 0 -- + exact/mate rows carry no real score target (see + build_training_tensors.py's score_probe_target) and must not pull + `raw` toward an arbitrary 0.0 placeholder.""" + return ((pred - target).pow(2) * valid).sum() -def nn_mse_sum(pred, target): - return (pred - target).pow(2).sum() +def masked_huber_sum(pred, target, valid, delta): + """Sum of Huber loss over only the rows where valid != 0 (same + masking as masked_mse_sum). Quadratic for |error| < delta + (preserves fine resolution on close, contested positions -- the + ones that actually matter for move choice), linear beyond it + (bounds how much a handful of large-magnitude blowout positions + can dominate the loss/gradient, unlike plain MSE). delta is in the + same pawn units as `pred`/`target`.""" + err = (pred - target).abs() + quad = 0.5 * err.pow(2) + lin = delta * (err - 0.5 * delta) + loss = torch.where(err < delta, quad, lin) + return (loss * valid).sum() def main(): @@ -164,16 +200,26 @@ def main(): "loss ratio between small and large WDL-space " "errors, pushing harder to fix decisively-wrong " "predictions. See powered_loss()'s docstring.") - ap.add_argument("--material-loss-weight", type=float, default=0.0, - help="weight on the auxiliary material_probe MSE " - "term (plain pawns^2, NOT WDL-space -- so this " - "is typically orders of magnitude larger in raw " + ap.add_argument("--score-loss-type", choices=["mse", "huber"], default="mse", + help="loss shape for the direct raw-vs-score term. " + "'mse': plain squared error (simple, but a " + "handful of large-magnitude blowout positions " + "can dominate it). 'huber': quadratic below " + "--huber-delta, linear beyond it -- bounds " + "blowout influence while keeping fine " + "resolution on close/contested positions.") + ap.add_argument("--huber-delta", type=float, default=1.0, + help="Huber loss transition point, in pawns -- " + "only used when --score-loss-type huber.") + ap.add_argument("--score-loss-weight", type=float, default=0.0, + help="weight on a direct MSE(raw, clipped Stockfish " + "score) term, straight on fc3's own output " + "(plain pawns^2, NOT WDL-space -- so this is " + "typically orders of magnitude larger in raw " "scale than the WDL loss; start small, e.g. " - "0.001-0.01, and check sanity_check_net.py " - "rather than assume a value works). 0.0 " - "(default) disables it entirely -- the probe " - "head still exists but gets no gradient, " - "equivalent to not having added it.") + "0.001, and check both holdout losses in " + "train_log.jsonl rather than assume a value " + "works). 0.0 (default) disables it entirely.") ap.add_argument("--threads", type=int, default=16, help="torch CPU thread count -- kept below this box's " "24 cores (not maxed out) since this is a shared " @@ -277,16 +323,17 @@ def main(): epoch_n = 0 t_epoch = time.time() - for w_idx, w_off, b_idx, b_off, stm, target, material in iterate_batches( + for w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid in iterate_batches( train_data, args.batch_size, shuffle=True, rng=rng): - w_idx, w_off, b_idx, b_off, stm, target, material = _to_device( - (w_idx, w_off, b_idx, b_off, stm, target, material), args.device) + w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid = _to_device( + (w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid), args.device) optimizer.zero_grad() - raw, material_pred = model.forward_with_material(w_idx, w_off, b_idx, b_off, stm) + raw = model.forward(w_idx, w_off, b_idx, b_off, stm) pred = torch.sigmoid(raw / args.k) wdl_loss = powered_loss(pred, target, args.loss_exponent) - material_loss = nn_mse_sum(material_pred, material) / len(target) - loss = wdl_loss + args.material_loss_weight * material_loss + score_loss = score_loss_fn(raw, score_val, score_valid, + args.score_loss_type, args.huber_delta) / len(target) + loss = wdl_loss + args.score_loss_weight * score_loss loss.backward() optimizer.step() @@ -296,34 +343,36 @@ def main(): global_step += 1 if global_step % args.eval_every_steps == 0: - holdout_wdl, holdout_material = evaluate( + holdout_wdl, holdout_score = evaluate( model, holdout_data, args.batch_size, args.k, - args.loss_exponent, args.device) + args.loss_exponent, args.device, + args.score_loss_type, args.huber_delta) elapsed = time.time() - t_start rec = { "step": global_step, "epoch": epoch, "train_mse_running": epoch_loss / max(epoch_n, 1), "holdout_mse": holdout_wdl, - "holdout_material_mse": holdout_material, + "holdout_score_probe_mse": holdout_score, "elapsed_s": round(elapsed, 1), "positions_seen": epoch_n + epoch * len(train_data), } print(f" step {global_step} epoch {epoch} " f"train_mse={rec['train_mse_running']:.5f} " f"holdout_mse={holdout_wdl:.5f} " - f"holdout_material_mse={holdout_material:.4f} " + f"holdout_score_probe_mse={holdout_score:.4f} " f"elapsed={elapsed:.0f}s") log_f.write(json.dumps(rec) + "\n") log_f.flush() - holdout_wdl, holdout_material = evaluate( + holdout_wdl, holdout_score = evaluate( model, holdout_data, args.batch_size, args.k, - args.loss_exponent, args.device) + args.loss_exponent, args.device, + args.score_loss_type, args.huber_delta) holdout_mse = holdout_wdl epoch_elapsed = time.time() - t_epoch print(f"Epoch {epoch} done: train_mse={epoch_loss / max(epoch_n, 1):.5f} " f"holdout_mse={holdout_wdl:.5f} " - f"holdout_material_mse={holdout_material:.4f} ({epoch_elapsed:.0f}s)") + f"holdout_score_probe_mse={holdout_score:.4f} ({epoch_elapsed:.0f}s)") ckpt_path = outdir / f"ckpt_epoch{epoch:03d}.pt" torch.save({ diff --git a/annotate_pgns/run_nnue_pipeline.py b/annotate_pgns/run_nnue_pipeline.py index 39bc487..6e51416 100644 --- a/annotate_pgns/run_nnue_pipeline.py +++ b/annotate_pgns/run_nnue_pipeline.py @@ -199,6 +199,21 @@ def main(): help="build_training_tensors.py's " "target = lambda*p_search + (1-lambda)*r_outcome " "for non-exact rows") + ap.add_argument("--k", type=float, default=None, + help="nnue_train.py's --k (the WDL-sigmoid scale, " + "pred=sigmoid(raw/k)) -- previously NOT passed " + "through at all, silently falling back to " + "nnue_train.py's own DEFAULT_K (1.3123, a Phase-A " + "placeholder), regardless of what K " + "fit_k_and_label.py actually fit for this corpus's " + "own p_search labels (stage 3 below) -- a real " + "label/loss K mismatch, fixed 2026-10-20. Required " + "now (no default) specifically so this can't go " + "silently unmatched again: read fit_k_and_label.py's " + "own printed 'Fitted K = ...' line for this corpus " + "and pass that (times whatever saturation-" + "correction factor empirical testing supports -- " + "see docs/NNUE.md's K-recalibration status).") ap.add_argument("--epochs", type=int, default=20) ap.add_argument("--patience", type=int, default=0, help="nnue_train.py's --patience (0 disables early " @@ -206,12 +221,21 @@ def main(): ap.add_argument("--batch-size", type=int, default=8192) ap.add_argument("--lr", type=float, default=1e-3) ap.add_argument("--loss-exponent", type=float, default=2.0) - ap.add_argument("--material-loss-weight", type=float, default=0.0) + ap.add_argument("--score-loss-weight", type=float, default=0.0) ap.add_argument("--force", action="store_true", help="re-run every stage even if its outputs " "already exist") args = ap.parse_args() + if args.k is None: + sys.exit("--k is required (no default, deliberately -- see its " + "help text): pass the K you actually want nnue_train.py's " + "loss to use. Do not guess -- run fit_k_and_label.py " + "first (or check its log from stage 3 below) to see what " + "it fits for THIS corpus, and apply whatever saturation-" + "correction factor has been validated empirically before " + "trusting its raw fit.") + pgn_paths = sorted({p for pattern in args.pgn_glob for p in glob.glob(pattern)}) if not pgn_paths: @@ -361,8 +385,8 @@ def main(): "--holdout-dir", tensors_holdout_dir, "--outdir", ckpt_dir, *resume_flag, "--epochs", args.epochs, "--batch-size", args.batch_size, - "--lr", args.lr, "--loss-exponent", args.loss_exponent, - "--material-loss-weight", args.material_loss_weight, + "--lr", args.lr, "--k", args.k, "--loss-exponent", args.loss_exponent, + "--score-loss-weight", args.score_loss_weight, "--threads", args.threads, "--device", args.device, "--patience", args.patience], args.force) @@ -372,10 +396,33 @@ def main(): run_stage( "export_nnue_weights.py", [args.outfile], [sys.executable, HERE / "export_nnue_weights.py", - ckpt, args.outfile, "--tensor-dir", tensors_train_dir], + ckpt, args.outfile, "--tensor-dir", tensors_train_dir, + "--holdout-tensor-dir", tensors_holdout_dir], + args.force) + + # 8. Refit the lazy-eval probe (nnue_vs_head.sh's NNUE_PROBE_WEIGHTS / + # command.c's "nnue probeload") against THIS checkpoint -- it's a + # separate offline ridge regression, not part of the exported .nnue + # file, and must be re-fit every generation (same rule as the + # calibration inside export_nnue_weights.py: not portable across + # checkpoints). Written alongside outfile, not into it. See + # fit_lazy_probe.py and the 2026-10-20 "keeping this stable across + # retrains" / lazy-eval conversation -- this is the regression fit + # only; the real search-node-traffic margin-safety calibration + # (CALIBRATE_MARGIN_SAFETY) is a separate, not-yet-automated step, + # deliberately not run here every generation (see that conversation + # for why it may not need to be). + probe_outfile = Path(args.outfile).with_name( + Path(args.outfile).stem + "_probe.bin") + run_stage( + "fit_lazy_probe.py", [probe_outfile], + [sys.executable, HERE / "fit_lazy_probe.py", + ckpt, probe_outfile, + "--train-tensor-dir", tensors_train_dir, + "--holdout-tensor-dir", tensors_holdout_dir], args.force) - print(f"\nPipeline complete: {args.outfile}") + print(f"\nPipeline complete: {args.outfile} (lazy probe: {probe_outfile})") if __name__ == "__main__": diff --git a/pgn/twic.pl b/pgn/twic.pl deleted file mode 100755 index 1fccae2..0000000 --- a/pgn/twic.pl +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/perl - -for ($x = 608; $x > 0; $x--) -{ - system "fetch http://www.chesscenter.com/twic/zips/twic" . $x . "g.zip"; -} diff --git a/src/chess.h b/src/chess.h index 3e65e7f..34099b2 100755 --- a/src/chess.h +++ b/src/chess.h @@ -1431,6 +1431,25 @@ typedef struct _COUNTERS UINT64 u64CyclesSuperLazyExit; UINT64 u64CyclesLazyExit; UINT64 u64CyclesFullEvalExit; + + // 2026-10-20: direct per-call latency of the two NNUE eval + // primitives (nnueeval.c), as opposed to the HCE-oriented + // whole-path-cost counters above (which NNUE's Eval() branch + // doesn't feed -- it's gated on g_fUseNnueEval at runtime, not + // #ifdef LAZY_EVAL). u64CyclesNnueProbeCall/u64CyclesNnueForwardCall + // time just the TIMED_EVAL_CALL-wrapped NnueProbeScore()/ + // NnueForward() calls themselves (eval.c's production NNUE + // branch); u64NnueProbeAttempts counts every probe call made + // (hit or miss) -- unlike u64LazyEvals, which only counts + // hits -- needed as the denominator for a true avg-cost-per- + // attempt (the question that actually matters for judging + // whether the gate pays for itself: cost is paid on every + // attempt, the saving only on a hit). u64FullEvals already + // counts NnueForward calls 1:1 on this path, so no separate + // attempt counter is needed for it. + UINT64 u64CyclesNnueProbeCall; + UINT64 u64CyclesNnueForwardCall; + UINT64 u64NnueProbeAttempts; } tree; @@ -2378,6 +2397,12 @@ extern CHAR g_rgfNnueFc3Weight[NNUE_L3]; extern float g_rgfNnueFc3Bias; extern float g_rgfNnueActScale[3]; extern float g_rgfNnueWeightScale[3]; +// Affine raw-to-real-score correction (a + b*raw), re-fit and baked +// into every export by CalibrateScoreMapping -- see nnueeval.c's +// declaration comment and the 2026-10-20 "keeping this stable across +// retrains" conversation. Defaults (0,1) are the identity transform. +extern float g_fNnueCalibA; +extern float g_fNnueCalibB; extern FLAG g_fNnueWeightsLoaded; extern FLAG g_fUseNnueEval; diff --git a/src/command.c b/src/command.c index 1767e98..9808c33 100644 --- a/src/command.c +++ b/src/command.c @@ -992,14 +992,22 @@ Return value: if (!STRCMPI(argv[1], "superlazyon")) { + // 2026-10-20: margin cp is now optional -- omit it to use + // eval.c's calibrated iNnueSuperLazyMarginByMatBucket table + // (the normal case); pass one to force a flat override across + // every material bucket instead, same convention as + // "nnue lazymargin". if (argc < 3) { - Trace("Usage: nnue superlazyon <margin cp>\n"); + g_iNnueSuperLazyMargin = -1; + g_fUseNnueSuperLazy = TRUE; + Trace("NNUE super-lazy (material-only) pre-filter enabled, " + "using calibrated per-bucket margin table.\n"); return; } g_iNnueSuperLazyMargin = (SCORE)atoi(argv[2]); g_fUseNnueSuperLazy = TRUE; - Trace("NNUE super-lazy (material-only) pre-filter enabled, margin=%d cp.\n", + Trace("NNUE super-lazy (material-only) pre-filter enabled, flat margin=%d cp.\n", g_iNnueSuperLazyMargin); return; } diff --git a/src/docs/NNUE.md b/src/docs/NNUE.md index dbc5d54..d1481c6 100644 --- a/src/docs/NNUE.md +++ b/src/docs/NNUE.md @@ -2690,3 +2690,410 @@ adding a new orchestrator script. architecture or export-format change -- so there's no reason to gate it behind the factorization/augmentation experiments above; can be added and swept independently. + +## Status (2026-10-20): losing-position score compression fixed (K +## recalibration), per-generation score calibration automated, lazy +## probe refit wired into the pipeline, margin-safety recalibrated +## against real search traffic, LOP-based promotion gate added + +Prompted by a direct observation: the production net's score in +clearly-losing positions barely distinguished "lost" (-2) from "about +to be mated" (-3) -- no meaningful magnitude gradation at the losing +extreme, despite real Stockfish scores in the same game ranging to +-13 or worse just before the actual mate. + +**Root cause, confirmed analytically and empirically**: `raw` (the +net's pre-sigmoid scalar) is only ever supervised through +`sigmoid(raw/K)` against a WDL-probability target -- never regressed +against a centipawn/pawn target directly. That sigmoid saturates hard +once a position is clearly decided; with this corpus's originally +fitted `K=2.0659`, `sigmoid(8/K)=0.9796` vs `sigmoid(13/K)=0.9982` -- +a target spread of under 0.02 for a 5-pawn difference in the real +score, leaving almost no training gradient to differentiate them. No +value of `lambda` (the `p_search`/`r_outcome` blend) fixes this, since +both blend candidates are already post-sigmoid probabilities. + +**What did NOT fix it, tried in order, each measured at a medium +scale (500k positions) before committing further**: +1. An auxiliary `score_probe` head (nnue_model.py, same mechanism as + `material_probe` -- a separate `Linear` on the shared accumulator, + trained with its own MSE loss against the real Stockfish score). + Modest, inconsistent improvement -- its gradient only ever reaches + the *shared embedding tables*, never `fc1`/`fc2`/`fc3`, so nothing + forces the actually-exported path to widen its range. +2. A direct `MSE(raw, clipped_score)` term added to the main loss + (replacing `score_probe` -- puts gradient straight on `fc1-3`). + Also modest/inconsistent across weights (0.001, 0.005) and loss + shapes (plain MSE vs. Huber) -- sometimes better on the winning + side, not reliably better on the losing side that actually + prompted this investigation. +3. **The actual fix: widen K itself.** Measured `raw` vs. real score + at two K values (2.07, 5.0) on the same held-out sample; the ratio + `slope/K` came out nearly identical at both points (0.3577 vs. + 0.3574) -- strong confirmation that `raw` scales linearly with K + as theory predicts, letting the right K be solved for directly: + `K* = 1/0.3575 ≈ 2.8`. Confirmed by actually training at K=2.8: + aggregate slope landed at 1.0005 (vs. 0.74 at the original fit), + with per-bucket magnitude tracking the real score far more closely + across the whole severity range, no overshoot the way K=5.0 + produced. Cross-checked against Stockfish's own documented NNUE + training write-up (`scaling_factor≈410cp=4.1 pawns`) -- our + corpus's own re-fit (2.8) came in lower, a reminder that this + project's corpus-specific fit isn't automatically wrong just for + disagreeing with a different engine's number, but also isn't + assumed right just for being "ours." +4. Also adopted: `--loss-exponent 2.5` (Stockfish's own documented + practice, already wired as `powered_loss()`'s `--loss-exponent`, + just not previously used) -- `|error|^2.5` instead of plain MSE in + WDL-probability space, which shrinks every error (since errors are + <1) but shrinks small ones proportionally more, concentrating + gradient on the worst-wrong predictions. Small, not dramatic, + improvement on top of the K fix; adopted since it's free. + +**Real residual tradeoff, not a flawless fix**: after K=2.8/exponent +2.5, aggregate calibration is essentially exact (slope ~1.0), but +there's genuine non-linearity left (overshoot at the extremes, +undershoot mid-range) and per-position noise went up slightly (mean +abs error 1.78 vs. 1.57 pawns on the full-scale retrain's held-out +sample) -- widening K to fix systematic compression also amplifies +whatever per-position imprecision the net already had, since the same +logit-space noise now maps to a bigger pawn-space error. Not treated +as disqualifying, but recorded so a future reader doesn't expect this +to be perfect. + +**Keeping this stable across future retrains -- the real answer isn't +"just reuse the same K"**: tested directly, holding K fixed within one +run still left measurable residual non-linearity/noise -- a fixed +*recipe* (K/lambda/loss-exponent) is not the same as a fixed *output +scale*, since training variance (data, convergence) shifts the +empirical raw-to-score mapping even with an identical recipe. Built +instead: +- `export_nnue_weights.py`'s new `CalibrateScoreMapping`: fits a fresh + affine correction `(a, b)` via OLS, `real_score ≈ a + b*raw`, + against the HOLDOUT tensor dir (not train -- an honest fit needs + positions the checkpoint didn't see), every single export. Baked + into a new file format version, `NNU4` (magic bump from `NNU3`): + two extra float32 fields, `CALIB_A`/`CALIB_B`, read by + `nnueeval.c`'s `NnueLoadWeights` and applied in both `NnueForward` + (`(CALIB_A + CALIB_B*fOut) * 100.0f`, replacing the old bare + `fOut * 100.0f`) and `NnueProbeScore` (same correction, since that + function's own doc comment promises it stays directly comparable to + `NnueForward`). This decouples engine search-tuning from needing the + raw net's internal scale to stay perfectly constant forever -- the + centipawn numbers search actually consumes stay calibrated by + construction, regenerated fresh every generation, regardless of how + much the underlying raw net drifts. +- Target for this fit is deliberately Stockfish's own score (the same + ground truth used throughout this project), not game outcome and + not HCE's scale -- outcome is noisier per-position by design + (lambda's whole point), and matching HCE's scale would just + reintroduce a version of the original compression problem in a + different flavor. +- **Real, previously-undiscovered bug found and fixed in the same + pass**: `run_nnue_pipeline.py` never threaded `--k` through to + `nnue_train.py` at all -- every prior pipeline-driven production run + (`sf_20261019`, `sf_20261020`, `sf98k`/the then-current production + net) silently trained its loss against `nnue_train.py`'s own + `DEFAULT_K=1.3123` (a Phase-A placeholder), completely independent + of whatever K `fit_k_and_label.py` actually fit for that corpus's + own `p_search` labels. `--k` is now a required argument to + `run_nnue_pipeline.py` (no silent default) specifically so this + can't happen again unnoticed. + +**Lazy-eval probe now refit automatically too**: new +`annotate_pgns/fit_lazy_probe.py` (ridge regression on the trained +net's own accumulator, targeting that same checkpoint's raw +`forward()` output pre-calibration, matching `NnueProbeScore`'s own +convention) wired into `run_nnue_pipeline.py` as stage 8, run +unconditionally after export -- every retrain now produces a fresh, +matching probe file alongside the `.nnue` weights, written to +`<outfile stem>_probe.bin`. The real search-node-traffic margin-safety +calibration (next item) is deliberately **not** automated the same +way -- too expensive to justify every generation, and not yet +established that it needs to be redone every generation either (see +below). + +**Full retrain done with the fixed recipe**: pooled all three +Stockfish self-play PGN files (`stockfish_self_play_training_data{,.0, +.1}.pgn`, ~120k games total), `lambda=0.99`, `K=2.8`, +`loss-exponent=2.5`, `score-loss-weight=0` (the score_probe/direct-term +experiments above were abandoned in favor of the K fix). 10,484,835 +training positions, 1,163,529 held out. Early-stopped at epoch 20 (5 +epochs past the best, epoch 15, per `--patience 5`), `holdout_mse= +0.00457`. Output: `annotate_pgns/runs/sf_20261020_recal/ +nnue_weights.nnue` (+ `nnue_weights_probe.bin`). + +**Margin-safety calibration re-run for real** (not the offline-sample +proxy that was all that existed before): 16 parallel shards of 12,500 +positions each (200,000 total) from `tests/cross_sectional_200k.ep_` +at `sd 6` -- same protocol as the original 2026-10-15 calibration, +re-run because that one was fit against an earlier net generation, +before this recalibration widened the net's raw output scale; the old +margins (`{180, 200, 190, 180, 170, 160, 150, 150}`) would have been +unsafe (too tight) against the new, wider-scale net's real swings. +New `iNnueLazyMarginByMatBucket` (p98 per material bucket, ~100-435M +real search-node swing samples per bucket): `{300, 320, 410, 640, 770, +770, 540, 230}`, now live in `eval.c`. New tooling: +`annotate_pgns/aggregate_marginsafety.py` sums the raw per-shard +histograms (`eval.c`'s `PersistMarginSafetyHistogramsToFile` format) +bucket-and-bin-wise and computes exact combined percentiles -- shard +percentiles don't combine validly on their own, only the raw counts +do. +- **Side finding**: NNUE super-lazy (material-only, no net at all) + measured much more competitive than the earlier "too wild to use" + verdict suggested -- p98 `{420, 450, 520, 730, 860, 850, 640, 290}`, + consistently ~30-40% looser than the real probe but the same order + of magnitude, not wildly worse. Likely explanation: the earlier + verdict was measured against the *old, uncalibrated* `NnueForward` + output, so comparing a stable-scale quantity (material) against a + compressed/wrong-scale one (the old net) spuriously inflated the + apparent disagreement. Not re-enabled by default -- `g_fUseNnueLazyProbe` + and `g_fUseNnueSuperLazy` both remain opt-in flags; flipping either + on by default is a strength/speed tradeoff decision that needs its + own match-based validation, not something to bundle into a + calibration-table update. +- **Open question, deliberately not resolved**: whether this expensive + live-search-traffic pass needs re-running every future generation, + or whether a cheap offline-residual proxy (same mechanism as + `CalibrateScoreMapping`, just also recording percentiles) tracks it + closely enough to trust between occasional live re-checks. Not yet + tested head-to-head; treat the live pass as the trusted source until + that comparison is actually done. + +**Validation: real, independent movement on the LOP benchmark** +(`tests/lop/` -- eval/move-agreement against a real Stockfish reference +on a fixed 10,000-position set, much cheaper than a match -- see that +section's own history below). Prior checkpoints (pre-recalibration): +87.089%/87.076% eval correlation, 46.800% move correlation -- +statistically indistinguishable from classic HCE's own 87.064%/46.830% +on the same benchmark. This recalibrated net: **90.319% eval +correlation, 48.550% move correlation** -- the first real movement on +this benchmark all session, and the basis on which this net was +promoted to production (see below). + +**Status correction on match results -- the "has not won a single +head-to-head match" note elsewhere in this document is now stale**: +against the *pre-recalibration* net, a real `sd 10`-matched self-play +run (`--sd 10`, isolating "is the net smarter per node" from the nps +handicap) produced LLR=+2.28 against an H1 bound of +2.94, score 0.635 +over 360 games (`-72 =119 +169`), climbing every update, halted early +as conclusive. NNUE now soundly beats classic HCE at matched search +depth -- this was not true when the adoption-criterion section below +was written. At matched real time (`--st 4`), the same +pre-recalibration net was close to even (score 0.447, LLR -0.37, +inconclusive, consistent with the known ~3.3x nps gap and +not-yet-recalibrated search margins costing it ~3 plies of depth) -- +**the recalibrated net in this status update has not been st4-tested +yet**; that is the planned next step, separate from this work. + +**Promotion policy changed, 2026-10-20**: `head_reference/nnue/ +promote.sh` now accepts LOP evidence (`--lop-baseline-log`/ +`--lop-candidate-log`, parsing `compare_lop`'s own stdout) as an +alternative to a match log, gated on the candidate's eval correlation +strictly exceeding the baseline's (no `--allow-negative` equivalent -- +if LOP doesn't even show improvement, it isn't evidence of anything). +Decision, not just convenience: requiring a full self-play match for +every incremental retrain (more data, same recipe) is impractical -- +a genuine few-Elo improvement is often not reliably resolvable in a +match without a very large game count, and this project's own history +(ECM solve-rate improving while early NNUE nets still lost every real +match, see below) already shows a cheap static benchmark isn't +*proof* of match strength either -- match evidence has its own noise +problems at this scale, it isn't pure ground truth being traded away +for convenience. LOP is accepted as the *routine* gate specifically +because requiring a match every time is the impractical alternative, +not because LOP has been shown to track match strength reliably yet. +An occasional real match is still recommended periodically to confirm +LOP-based promotions haven't quietly drifted away from actual playing +strength -- this recalibrated net was itself promoted on LOP evidence +alone (see `head_reference/nnue/current/metadata.json`), with a real +match against the previous baseline still outstanding. + +## Status (2026-10-20, continued): lazy probe measured net-negative on +## real search traffic, super-lazy confirmed a clean win and now +## defaults on + +Follow-up to the "Engine calibration checklist" section above, picking +up item 1's lazy-eval fast path -- the goal was to actually turn the +two gates on (both were built and opt-in per the recalibration section +just above) and measure real before/after numbers, not just trust the +earlier margin-safety calibration as sufficient justification on its +own. + +**Methodology**: `ecm_ringers`/`ecm_confident_quick`/`ecm_hard_quick` +(the three curated suites), each at both `sd 10` (fixed depth) and +`sn 5000000` (fixed node budget), run as a clean baseline (gates off), +then with only the lazy probe on, then with only super-lazy on, all +against the same `eval.nnue` weights -- never mixed/parallel-vs- +sequential in a way that could introduce load-noise into the +comparison (confirmed box load ~4.3 on a lightly-loaded multi-core +machine throughout; `sd`/`sn` hold depth/node-budget fixed by +construction regardless of load, so nps and solve-count differences +here are real, not noise). + +**Baseline** (gates off): 108/191 solved across all three suites at +`sd 10` (7/11, 73/90, 27/90), 117/191 at `sn 5000000` (8/11, 78/90, +31/90); nps 479k-539k depending on suite/mode. + +**Lazy probe alone, at the real p98 margin table (`{300, 320, 410, +640, 770, 770, 540, 230}` by material bucket, from the real +`cross_sectional_200k.ep_`/16-shard run)**: a clear net LOSS -- +**nps down 3-7% in every configuration**, and fewer solves in 4 of 6 +configs (ringers 6/11 and 7/11 vs baseline's 7/11 and 8/11; +confident_quick 70/90 flat/down; hard_quick 24/90 and 28/90 vs 27/90 +and 31/90). Not explained by the probe being expensive -- see the +latency measurement below. + +**Direct per-call latency measurement** (new instrumentation, piled +into the existing `EVAL_TIME` build profile rather than a new flag -- +`TIMED_EVAL_CALL` wraps the `NnueProbeScore()`/`NnueForward()` call +sites in `eval.c`'s production NNUE branch, accumulating into two new +always-present-but-only-`EVAL_TIME`-populated `COUNTERS.tree` fields, +`u64CyclesNnueProbeCall`/`u64CyclesNnueForwardCall`, plus a new +`u64NnueProbeAttempts` counter -- unlike `u64LazyEvals`, which only +counts *hits*, this counts every attempt, hit or miss, since the gate +pays its cost on every attempt and only recoups it on a hit). +Reported in `root.c` right after the (NNUE-irrelevant, classic-HCE- +only) `LAZY_EVAL` exit-breakdown block, with the measurement's own +~38-cycle rdtsc-pair overhead printed alongside rather than silently +subtracted, so the raw numbers can be sanity-checked against it: + +``` +NnueProbeScore: ~272 cyc/call (net ~235 after overhead) +NnueForward: ~4100 cyc/call (net ~4060 after overhead) +``` + +**A clean 15:1 ratio, extremely consistent across all 11 `ecm_ringers` +positions (272±3 cyc every time)** -- the probe itself is cheap and +behaving exactly as designed. Implied breakeven hit rate: `272/4100 ≈ +6.6%`. The docs' earlier 2026-10-16 measurement (before the K/lambda +rescale) claimed a 14-23% real hit rate, comfortably above breakeven. + +**But the real hit rate on this suite, at the current (post-rescale) +p98 table, is ~1.0% overall** (71,022 hits / 7,100,023 attempts on +`ecm_ringers`, per-position rates ranging 0.03%-6.98%) -- an order of +magnitude under breakeven. Expected per-call cost with the gate on: +`272 + 0.99*4100 ≈ 4331` cyc vs. `4100` cyc with it off -- a ~5.6% +per-call slowdown, matching the measured nps regression almost +exactly. **Root cause, not just a symptom**: the p98 margin table is +calibrated purely for *safety* (bounding `|NnueForward() - +NnueProbeScore()|` error to the 98th percentile), a completely +different statistic from *how close typical nodes sit to their own +alpha/beta window*, which is what actually determines hit rate. A +margin can be a perfectly correct p98 safety bound and still almost +never let the gate fire if real windows at that material bucket are +typically narrower than the error the probe needs room for -- which +is apparently true for most of this table's buckets (only bucket 7's +230 happens to land in useful territory; the large buckets, 640-770, +are far out past where any real window would resolve a cutoff). + +**Empirical margin-vs-hit-rate sweep** (flat override across all +buckets via `nnue lazymargin <cp>`, `ecm_ringers`/`sd 10` only -- fast +enough to run live rather than needing a real calibration pass): + +| flat margin | hit rate | solved/11 | nps | +|---|---|---|---| +| 770 | 0.00% | 6 | 509k | +| 400 | 1.80% | 6 | 510k | +| 230 | 16.0% | 6 | 538k | +| 150 | 34.9% | 5 | 582k | +| 100 | 45.0% | 4 | 632k | +| 70 | 61.6% | 4 | 696k | +| 50 | 67.9% | 3 | 775k | +| 30 | 78.9% | 2 | 896k | +| 15 | 87.9% | 2 | 1024k | + +Smooth tradeoff curve, no cliff -- the knee sits around margin≈200-230, +where nps is clearly improving (538k, a real win over baseline) with +no solve cost yet; solves start falling steadily from 150 down. This +is genuine evidence a per-bucket table *could* be built to win cleanly +-- it just isn't the p98 table, which was never tuned for this +objective at all. + +**p95 recalibration attempt -- inconclusive, not a real retest**: +tried to avoid a full hour+ `cross_sectional_200k.ep_` re-collection by +recomputing percentiles from already-persisted histogram data. Found +`eval.c`'s `PersistMarginSafetyHistogramsToFile` writes to +`/var/tmp/typhoon/marginsafety_raw_histograms.txt` by default, and +`annotate_pgns/aggregate_marginsafety.py` keeps the full binned +histogram (not just the percentiles it prints), so an arbitrary +percentile is cheap to recompute *if the real per-shard files still +exist*. **They don't** -- only one small leftover file was found +(~1.9M total `nnue_lazy` samples across buckets vs. the real run's +~100-435M/bucket; almost certainly one shard or an early smoke test, +not the aggregated dataset), and a project-wide `find -name "*hist*"` +turned up nothing else NNUE-related (one old, unrelated classic-HCE +residual-calibration file from a `guess2_30k` run). Computed p95 from +this small/wrong-provenance file anyway as a fast, low-confidence +stand-in (`{180, 240, 520, 530, 670, 720, 480, 310}` by bucket) and +re-ran all three suites, both depths, with the lazy probe on this +table. **Result was noisier than the p98 run, not clearly better**: +some solve counts recovered (ringers back to 8/11 at `sn5M`, +hard_quick up to 26/90 and 33/90), but nps actually *dropped* in +several configs relative to p98 (`sn5M` confident_quick 504k vs p98's +537k; hard_quick 483k vs 512k) -- backwards from what a looser margin +should do, strongly suggesting the small sample's per-bucket numbers +don't reliably reflect the real distribution. **Not trusted as a real +retune** -- a legitimate hit-rate-targeted per-bucket calibration +still needs the full `cross_sectional_200k.ep_` collection (or +equivalent), this time explicitly measuring node-level alpha/beta +proximity (what determines hit rate), not just probe-vs-forward error +(what the existing harness measures, which only bounds safety). + +**Super-lazy alone, at the real p98 margin table (newly built this +session -- previously a single flat runtime override with no +per-bucket fit at all, `g_iNnueSuperLazyMargin`)**: measured a clean +win across the board. `sd 10`: ringers 5/11 (down from 7, but ringers +is only 11 positions -- noisy), confident_quick 70/90 (down from 73, +flat vs lazy's own 70), hard_quick 26/90 (close to baseline's 27); +nps 478k-504k, roughly matching or beating baseline (479k-502k). +`sn 5000000`: ringers 6/11 (down from 8), confident_quick **79/90** +(up from 78), hard_quick **33/90** (up from 31); nps **540k-601k**, +a genuine 1-12% win over baseline (514k-539k) in every suite. Net +read: real nps win, solve-count effect mixed but mostly neutral-to- +positive outside the small `ecm_ringers` suite (where single-position +flips swing the ratio a lot) -- per the user's own judgment, ECM +solve-count noise at this suite size isn't treated as a correctness +signal worth chasing into `tests/ECM.md` lookups; the nps win is the +real, load-bearing result. + +**Same-bucketing confirmed**: super-lazy's own margin-safety +calibration (`RecordNnueSuperLazyMarginSafetySwing`, +`g_uNnueSuperLazyMarginSwingMax[BASE_MARGIN_MAT_BUCKETS]` etc.) uses +the identical `_MaterialBucket()` (8 buckets, combined-army-scaler +width 8) as the lazy probe's own table. + +**No bucket excluded outright** (unlike classic HCE's own super-lazy, +which structurally excludes material bucket 0 -- bare-king/pawn- +dominated endgames -- per a specific *exceeded-rate* finding, 6.8% +blow-through from pawn-structure blindness, since HCE's super-lazy +runs before `_EvalPawns` entirely). NNUE's measured p98 swing by +bucket, `{420, 450, 520, 730, 860, 850, 640, 290}`, has bucket 0 as +the *second-lowest* value, not an outlier -- nothing in the data says +to disable it. Caveat: no exceeded-rate-specific check (as opposed to +plain p98 magnitude) has been run for NNUE's super-lazy the way it was +for HCE's, so this is "nothing visible says disable a bucket," not +"confirmed safe at every bucket." + +**Decision**: lazy probe gate code kept live (cheap to leave dormant, +and the latency math + margin sweep both say a real win is plausible +with the right margins -- just not demonstrated yet), but +`g_fUseNnueLazyProbe` stays `FALSE` by default (`nnueeval.c`), margin +table reverted to the real p98 values (the p95 guess is preserved only +as a commented-out, documented-as-unreliable alternative). Super-lazy +promoted to default-on (`g_fUseNnueSuperLazy = TRUE`, `nnueeval.c`), +using its own new per-bucket `iNnueSuperLazyMarginByMatBucket` table +(`eval.c`) rather than the old flat-only override, which remains +available via `g_iNnueSuperLazyMargin >= 0` for experimentation. Both +gates only take effect when `g_fUseNnueEval` is also on. + +**Follow-up, not yet done**: a real hit-rate-targeted per-bucket lazy +probe retune, against ordinary (non-adversarial-tactical) search +traffic such as `twic_sample.ep_` rather than ECM -- the current +margin-vs-hit-rate sweep and the docs' original 14-23%-hit-rate/ +10.7%-nps-win measurement were on different corpora, and ECM's +narrower forcing-line alpha/beta windows may themselves suppress hit +rate in a way that doesn't generalize to normal play. Also worth +measuring: an exceeded-rate check for NNUE super-lazy's bucket 0, +matching the specific diagnostic that led HCE to exclude it, rather +than relying on p98 magnitude alone. @@ -7239,23 +7239,86 @@ Return value: if ((TRUE == g_fUseNnueEval) && (TRUE == g_fNnueWeightsLoaded)) { // p98 of |NnueForward() - NnueProbeScore()| per material - // bucket. Superseded 2026-10-15: originally derived from a - // 91M-sample population over three curated tactical suites - // (sd 10) -- replaced same day by a far larger and more - // representative 3.09-BILLION-sample population, 16 parallel - // shards of 12,500 positions each (200,000 total) from - // tests/cross_sectional_200k.ep_ at sd 6. Per-suite tactical - // positions are a narrow, biased slice (selected for sharp - // tactics); the cross-sectional pool's much broader position - // variety surfaced real outlier swings the narrower sample - // never saw (max up to 534cp here vs ~472cp before) while - // landing close to the same per-bucket numbers overall (mostly - // 0-10cp different) -- see docs/NNUE.md's "Lazy-evaluation - // tiering plan". Same bucketing as _MaterialBucket (combined - // army scaler, width BASE_MARGIN_MAT_WIDTH). + // bucket. Superseded 2026-10-20: the previous table (180, 200, + // 190, 180, 170, 160, 150, 150) was fit against an earlier net + // generation, before that day's K/lambda/loss-exponent + // recalibration (see docs/NNUE.md's "keeping this stable + // across retrains" conversation) widened the net's raw output + // scale to actually track Stockfish's real score magnitude -- + // the old, narrower-scale margins would have been unsafe (too + // tight) against the new, wider-scale net. Re-measured the + // same way (16 parallel shards of 12,500 positions each, + // 200,000 total, from tests/cross_sectional_200k.ep_ at sd 6, + // ~100-435M swing samples per bucket) against the recalibrated + // net (annotate_pgns/runs/sf_20261020_recal/nnue_weights.nnue) + // and its freshly-refit probe -- see + // annotate_pgns/fit_lazy_probe.py and aggregate_marginsafety.py. + // Must be re-measured again for any future net whose raw + // output scale changes meaningfully (a new K, a materially + // different training recipe) -- not assumed to stay valid + // forever just because the recipe nominally stayed the same + // (see that same conversation for why). Same bucketing as + // _MaterialBucket (combined army scaler, width + // BASE_MARGIN_MAT_WIDTH). + // 2026-10-20 status: lazy probe gate measured net NEGATIVE on + // real ECM search traffic at this margin table (p98, from the + // real cross_sectional_200k.ep_/16-shard run) -- hit rate ~1% + // overall, far under the ~6.6% breakeven implied by + // NnueProbeScore's own ~272cyc vs NnueForward's ~4100cyc + // latency (measured directly, see root.c's EVAL_TIME-gated + // "NNUE eval primitive latency" report). A p95 guess (NOT from + // this real run -- its per-shard raw histograms no longer + // exist on disk; computed instead from a much smaller + // leftover file of unknown/mismatched provenance, see git + // history) didn't clearly fix it either, and came back noisier + // than p98 on nps in several configs. g_fUseNnueLazyProbe + // stays FALSE by default (nnueeval.c) pending a real + // hit-rate-targeted per-bucket retune (not yet done -- see + // docs/NNUE.md) measured against ordinary search traffic + // (twic_sample.ep_-like), not just ECM's narrower tactical + // windows. Code kept live (not deleted) since the mechanism is + // cheap to leave dormant and the latency math says a real win + // is plausible with the right margins -- just not demonstrated + // yet. This table is the one real, validated data point (p98, + // safety-only objective) and is what a future retune should + // start from and compare against. static const SCORE iNnueLazyMarginByMatBucket[BASE_MARGIN_MAT_BUCKETS] = { - 180, 200, 190, 180, 170, 160, 150, 150, + 300, 320, 410, 640, 770, 770, 540, 230, + }; + // p95 experiment (2026-10-20) -- see the status comment above + // for why this isn't trusted as a real calibration. Kept here + // for reference/reproducibility, not as a ready-to-flip + // alternative. + // static const SCORE iNnueLazyMarginByMatBucketP95Guess[BASE_MARGIN_MAT_BUCKETS] = + // { + // 180, 240, 520, 530, 670, 720, 480, 310, + // }; + + // 2026-10-20: NNUE super-lazy's own per-bucket table, same + // calibration run (cross_sectional_200k.ep_, sd 6, same + // _MaterialBucket) as iNnueLazyMarginByMatBucket above -- + // p98 of |NnueForward() - pos->iMaterialBalance[uToMove]|. + // Previously a single flat runtime override + // (g_iNnueSuperLazyMargin) with no per-bucket fit at all; per + // docs/NNUE.md's "Side finding" (2026-10-20), NNUE super-lazy's + // real measured swing is only ~30-40% looser than the real + // probe's, same order of magnitude, not "too wild to use" as + // an earlier (differently-scaled-net) measurement had + // suggested. No bucket excluded outright the way classic + // HCE's super-lazy excludes material bucket 0 (eval.c's + // fBareKingEitherSide/uSuperLazyMatBucket==0 guard) -- + // that exclusion was driven by a specific *exceeded-rate* + // finding (6.8% blow-through from pawn-structure blindness, + // since HCE's super-lazy runs before _EvalPawns), not by p98 + // magnitude alone, and NNUE's bucket 0 p98 here (420) is in + // fact the second-lowest of the 8, not an outlier. No + // equivalent exceeded-rate check has been run for NNUE's + // super-lazy yet, though -- this is "nothing visible says + // disable a bucket," not "confirmed safe at every bucket." + static const SCORE iNnueSuperLazyMarginByMatBucket[BASE_MARGIN_MAT_BUCKETS] = + { + 420, 450, 520, 730, 860, 850, 640, 290, }; #ifdef CALIBRATE_MARGIN_SAFETY @@ -7307,20 +7370,27 @@ Return value: // entirely -- this is the actual speed win. Otherwise (gate // disabled, probe not loaded, or inconclusive), fall through // to the full pass exactly as before this feature existed. - // Super-lazy pre-filter, experimental -- 2026-10-16: cheapest - // possible check, pure material balance, no net evaluation at - // all. Previously measurement-only (CALIBRATE_MARGIN_SAFETY - // above records the real swing but always used an infinite - // sentinel margin, i.e. never actually gated). Deliberately - // loose/uncalibrated here per the "safety doesn't win games" - // experiment -- g_iNnueSuperLazyMargin is a flat runtime - // override (see nnueeval.c), not a per-bucket fit. + // Super-lazy pre-filter -- cheapest possible check, pure + // material balance, no net evaluation at all. Previously + // measurement-only (CALIBRATE_MARGIN_SAFETY above records the + // real swing but always used an infinite sentinel margin, i.e. + // never actually gated), then briefly a single uncalibrated + // flat runtime override. 2026-10-20: now a real per-material- + // bucket table (iNnueSuperLazyMarginByMatBucket, above), + // calibrated the same way/same run as the lazy probe's own + // table -- g_iNnueSuperLazyMargin remains available as an + // explicit flat override (>= 0) for experimentation, matching + // g_iNnueLazyMarginOverride's convention; < 0 (the default) + // uses the calibrated table. if (TRUE == g_fUseNnueSuperLazy) { SCORE iSuperLazyScore = pos->iMaterialBalance[pos->uToMove]; + SCORE iSuperLazyMargin = (g_iNnueSuperLazyMargin >= 0) ? + g_iNnueSuperLazyMargin : + iNnueSuperLazyMarginByMatBucket[_MaterialBucket(pos)]; - if (((iSuperLazyScore - g_iNnueSuperLazyMargin) > iBeta) || - ((iSuperLazyScore + g_iNnueSuperLazyMargin) < iAlpha)) + if (((iSuperLazyScore - iSuperLazyMargin) > iBeta) || + ((iSuperLazyScore + iSuperLazyMargin) < iAlpha)) { // Feed the same aggregate counters root.c's "Eval exit // breakdown" line reads -- otherwise that line stays @@ -7339,7 +7409,10 @@ Return value: } if ((TRUE == g_fUseNnueLazyProbe) && (TRUE == g_fNnueProbeLoaded)) { - SCORE iProbeScore = NnueProbeScore(ctx); + SCORE iProbeScore; + INC(ctx->sCounters.tree.u64NnueProbeAttempts); + TIMED_EVAL_CALL(ctx, u64CyclesNnueProbeCall, + iProbeScore = NnueProbeScore(ctx)); SCORE iMargin = (g_iNnueLazyMarginOverride >= 0) ? (SCORE)g_iNnueLazyMarginOverride : iNnueLazyMarginByMatBucket[_MaterialBucket(pos)]; @@ -7359,7 +7432,9 @@ Return value: } } { - SCORE iNnueScore = NnueForward(ctx); + SCORE iNnueScore; + TIMED_EVAL_CALL(ctx, u64CyclesNnueForwardCall, + iNnueScore = NnueForward(ctx)); INC(ctx->sCounters.tree.u64FullEvals); ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_FULL; g_u64NnueFullEvalsByBucket[_MaterialBucket(pos)]++; diff --git a/src/nnueeval.c b/src/nnueeval.c index 7caadef..583c2db 100644 --- a/src/nnueeval.c +++ b/src/nnueeval.c @@ -45,6 +45,24 @@ CHAR g_rgfNnueFc3Weight[NNUE_L3]; float g_rgfNnueFc3Bias; float g_rgfNnueActScale[3] = { 1.0f, 1.0f, 1.0f }; float g_rgfNnueWeightScale[3] = { 1.0f, 1.0f, 1.0f }; +// Affine correction so the engine's consumed centipawn output tracks +// Stockfish's own real score, not the net's nominal raw WDL-sigmoid +// scale -- see export_nnue_weights.py's CalibrateScoreMapping and the +// 2026-10-20 "keeping this stable across retrains" conversation. A +// fixed K/lambda/loss-exponent training recipe is NOT enough on its +// own to guarantee the search's consumed cp scale stays constant +// across future retrains -- training variance (data, convergence) +// still shifts the empirical raw-to-score mapping even with an +// identical recipe. Re-fitting and re-baking this correction into +// every export, rather than trusting a fixed multiplier, is what +// actually decouples engine-side search tuning from needing the raw +// net's internal scale to stay perfectly constant forever. Defaults +// (0,1) are the identity transform -- the pre-2026-10-20 behavior -- +// so an old build loading a newer file (or vice versa, before the +// dimension check below even runs) degrades to the old bare-*100 +// convention rather than silently miscalibrating. +float g_fNnueCalibA = 0.0f; +float g_fNnueCalibB = 1.0f; FLAG g_fNnueWeightsLoaded = FALSE; FLAG g_fUseNnueEval = FALSE; @@ -86,12 +104,13 @@ Return value: } if ((1 != fread(szMagic, 4, 1, fp)) || - (0 != memcmp(szMagic, "NNU3", 4))) + (0 != memcmp(szMagic, "NNU4", 4))) { Trace("NnueLoadWeights: %s is not a valid NNUE weights file " - "(bad magic -- expected \"NNU3\"; an older \"NNU1\"/" - "\"NNU2\" file predates the FC head's int8 quantization " - "and is no longer readable)\n", szFilename); + "(bad magic -- expected \"NNU4\"; an older \"NNU1\"/" + "\"NNU2\"/\"NNU3\" file predates the raw-to-score " + "calibration fields and is no longer readable)\n", + szFilename); fclose(fp); return FALSE; } @@ -130,6 +149,14 @@ Return value: fclose(fp); return FALSE; } + if ((1 != fread(&g_fNnueCalibA, sizeof(g_fNnueCalibA), 1, fp)) || + (1 != fread(&g_fNnueCalibB, sizeof(g_fNnueCalibB), 1, fp))) + { + Trace("NnueLoadWeights: %s truncated (score calibration)\n", + szFilename); + fclose(fp); + return FALSE; + } #define READ_OR_FAIL(ptr, elsize, count) \ do { \ @@ -265,7 +292,7 @@ Return value: g_rgfNnueFc3Bias; } - return (SCORE)(fOut * 100.0f); + return (SCORE)((g_fNnueCalibA + g_fNnueCalibB * fOut) * 100.0f); } @@ -420,14 +447,23 @@ FLAG g_fUseNnueLazyProbe = FALSE; // docs/NNUE.md "paradox: safety doesn't win games" discussion. // g_iNnueLazyMarginOverride < 0 means "use eval.c's calibrated // per-material-bucket table as before"; >= 0 means "use this flat cp -// value for every bucket instead". g_fUseNnueSuperLazy/ -// g_iNnueSuperLazyMargin gate a brand new, even cheaper pre-filter -// (pure material balance, no net at all) ahead of the probe -- this -// was previously measurement-only (see eval.c's CALIBRATE_MARGIN_SAFETY -// block) and is now wired into the real Eval() path for the first time. +// value for every bucket instead". g_fUseNnueSuperLazy gates an even +// cheaper pre-filter (pure material balance, no net at all) ahead of +// the probe. g_iNnueSuperLazyMargin follows the identical convention +// as of 2026-10-20 (was a flat-only override before that date, with no +// per-bucket fit at all): < 0 means "use eval.c's calibrated +// iNnueSuperLazyMarginByMatBucket table", >= 0 means "use this flat cp +// value for every bucket instead". INT g_iNnueLazyMarginOverride = -1; -FLAG g_fUseNnueSuperLazy = FALSE; -SCORE g_iNnueSuperLazyMargin = 0; +// 2026-10-20: super-lazy defaults ON -- measured a clean win (nps up, +// no consistent solve-rate cost) across all three curated ECM suites +// at both sd 10 and sn 5000000, using the calibrated per-bucket +// iNnueSuperLazyMarginByMatBucket table (eval.c), unlike the lazy +// probe gate just above, which stays off by default (measured net +// negative -- see its own table's comment in eval.c for the full +// story). Still only takes effect when g_fUseNnueEval is also on. +FLAG g_fUseNnueSuperLazy = TRUE; +SCORE g_iNnueSuperLazyMargin = -1; FLAG @@ -554,5 +590,11 @@ Return value: fSum += v * g_rgfNnueProbeWeight[NNUE_ACCUMULATOR_SIZE + i]; } - return (SCORE)(fSum * 100.0f); + // fSum approximates NnueForward's pre-calibration raw fOut (it was + // fit by ridge regression against exactly that target -- see + // docs/NNUE.md's "Lazy-evaluation tiering plan") -- apply the same + // (g_fNnueCalibA, g_fNnueCalibB) correction so this stays directly + // comparable to NnueForward's actual output, per this function's + // own doc comment above. + return (SCORE)((g_fNnueCalibA + g_fNnueCalibB * fSum) * 100.0f); } @@ -628,6 +628,59 @@ Return value: #endif } #endif +#ifdef EVAL_TIME + // 2026-10-20: NNUE's own per-call latency, independent of the + // LAZY_EVAL block above (NNUE is a runtime g_fUseNnueEval switch, + // not a #ifdef LAZY_EVAL path, so it never fed that block's + // counters). Direct answer to "is the lazy-probe gate paying for + // itself": the gate costs u64CyclesNnueProbeCall on EVERY attempt + // (hit or miss -- u64NnueProbeAttempts, not just u64LazyEvals' + // hit count) and only SAVES a full-forward call on a hit, so the + // gate is worth it only if (hit rate) * (forward cost - probe + // cost) exceeds (miss rate) * (probe cost). + // + // Caveat printed alongside rather than silently baked in: each + // TIMED_EVAL_CALL is a pair of back-to-back SystemReadTimeStamp- + // Counter() (rdtsc) reads, and rdtsc itself isn't free (commonly + // cited around ~80 cycles, more with the serializing behavior + // some cores give it) -- for NnueForward's thousands of cycles + // that overhead is noise, but for NnueProbeScore's much smaller + // 512-wide dot product it may not be. Measure the pair's own + // overhead right here (an empty back-to-back read, no call in + // between) so the raw numbers below can be read against it rather + // than taken as exact. + { + UINT64 u64OverheadStart = SystemReadTimeStampCounter(); + UINT64 u64OverheadEnd = SystemReadTimeStampCounter(); + UINT64 u64PairOverhead = u64OverheadEnd - u64OverheadStart; + UINT64 u64ProbeCyc = ctx->sCounters.tree.u64CyclesNnueProbeCall; + UINT64 u64ProbeAttempts = ctx->sCounters.tree.u64NnueProbeAttempts; + UINT64 u64FwdCyc = ctx->sCounters.tree.u64CyclesNnueForwardCall; + UINT64 u64FwdCalls = ctx->sCounters.tree.u64FullEvals; + double dProbeAvg = u64ProbeAttempts ? + (double)u64ProbeCyc / (double)u64ProbeAttempts : 0.0; + double dFwdAvg = u64FwdCalls ? + (double)u64FwdCyc / (double)u64FwdCalls : 0.0; + + if ((0 != u64ProbeAttempts) || (0 != u64FwdCalls)) + { + Trace("NNUE eval primitive latency (rdtsc pair overhead ~%" + COMPILER_LONGLONG_UNSIGNED_FORMAT" cyc, subtract for a " + "rough net estimate):\n" + " NnueProbeScore: %10.1f cyc/call (raw, %" + COMPILER_LONGLONG_UNSIGNED_FORMAT" attempts, net ~%10.1f)\n" + " NnueForward: %10.1f cyc/call (raw, %" + COMPILER_LONGLONG_UNSIGNED_FORMAT" calls, net ~%10.1f)\n", + u64PairOverhead, + dProbeAvg, u64ProbeAttempts, + (dProbeAvg > (double)u64PairOverhead) ? + (dProbeAvg - (double)u64PairOverhead) : 0.0, + dFwdAvg, u64FwdCalls, + (dFwdAvg > (double)u64PairOverhead) ? + (dFwdAvg - (double)u64PairOverhead) : 0.0); + } + } +#endif Trace("Extensions: (%u +, %u q+, %u 1mv, %u !kmvs, %u mult+, %u pawn\n" " %u threat, %u zug, %u sing, %u endg, %u bm, %u recap)\n" " check by amount (1/4,1/2,3/4,1 ply): %u %u %u %u\n" |
