NNUE: re-fit raw-to-score calibration, NNU4 weight format, super-lazy default-on, lazy probe measured net-negative

Fits an affine (a + b*raw) correction into every exported .nnue file
(export_nnue_weights.py's CalibrateScoreMapping) so the engine's
consumed centipawn output tracks Stockfish's real score scale rather
than the net's nominal raw WDL-sigmoid output -- training variance
alone shifts this mapping across retrains even with an identical
recipe, so it's re-fit and baked in per export rather than assumed
constant. Weight format bumped to "NNU4" to carry the two new floats;
older NNU1/2/3 files are no longer readable.

Super-lazy (material-only) pre-filter promoted to default-on
(g_fUseNnueSuperLazy = TRUE) with a recalibrated per-material-bucket
margin table (measured against the recalibrated net, 16-shard
cross_sectional_200k.ep_ sample) -- real, measured nps win across all
three curated suites. The separate lazy-probe gate (accumulator-based
early exit) is measured net-negative on real search traffic at this
net's probe-vs-forward latency ratio and stays off by default
(g_fUseNnueLazyProbe = FALSE); code kept live since margin tuning
could still flip this, per docs/NNUE.md.

Also: direct per-call latency instrumentation for NnueProbeScore/
NnueForward (root.c, EVAL_TIME build) to make the lazy-probe
breakeven math measurable directly instead of inferred; QSearch
futility margins given real-data-derived NNUE_FULL/NNUE_LAZY
starting points.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

1 files changed, 38 insertions, 26 deletions

diff --git a/annotate_pgns/build_training_tensors.py b/annotate_pgns/build_training_tensors.py
index 0f69d6c..ff9e772 100644
--- a/annotate_pgns/build_training_tensors.py
+++ b/annotate_pgns/build_training_tensors.py
@@ -51,16 +51,14 @@ import numpy as np
 
 from nnue_feature_spec import active_feature_indices, chess
 
-# Standard piece values for the auxiliary material-probe target
-# (nnue_model.py's material_probe, added 2026-10-17 after a measured
-# failure -- see that module's docstring). Plain material count,
-# mover-relative, NOT run through the WDL sigmoid -- this is a
-# straight regression target, deliberately non-saturating. Kings
-# excluded (always 1-1 on a legal board, carries no signal).
-_PIECE_VALUES = {
-    chess.PAWN: 1, chess.KNIGHT: 3, chess.BISHOP: 3,
-    chess.ROOK: 5, chess.QUEEN: 9,
-}
+# Clip bound for the auxiliary score-probe target (nnue_model.py's
+# score_probe, added 2026-10-20 -- see that module's docstring). A
+# handful of blowout-but-not-forced-mate scores (see the user's own
+# +13.67-before-mate example) would otherwise dominate the probe's
+# MSE; exact/mate rows are excluded entirely (see SCORE_VALID below),
+# not just clipped, since a mate score isn't "the Stockfish score" in
+# the pawn sense this probe targets.
+SCORE_CLIP_PAWNS = 20.0
 
 
 def position_target(pos, lam):
@@ -78,14 +76,18 @@ def position_target(pos, lam):
     return lam * p_search + (1.0 - lam) * r_outcome
 
 
-def material_balance(board):
-    """Mover-relative material balance in pawns, +ve = side to move
-    is materially ahead."""
-    total = 0
-    for piece_type, value in _PIECE_VALUES.items():
-        total += value * len(board.pieces(piece_type, chess.WHITE))
-        total -= value * len(board.pieces(piece_type, chess.BLACK))
-    return total if board.turn == chess.WHITE else -total
+def score_probe_target(pos):
+    """Returns (score, valid) for the auxiliary score-probe target --
+    Stockfish's own raw per-position score (pos["score"], already
+    mover-relative pawns per extract_selfplay_positions.py), clipped
+    to +-SCORE_CLIP_PAWNS, or (0.0, False) for exact/mate rows (no
+    "score" field, or not a real Stockfish-judged centipawn-ish value
+    -- same exclusion reasoning as p_search's, see NNUE.md)."""
+    if pos.get("exact") or "score" not in pos:
+        return 0.0, False
+    s = pos["score"]
+    s = max(-SCORE_CLIP_PAWNS, min(SCORE_CLIP_PAWNS, s))
+    return s, True
 
 
 def iter_positions(path):
@@ -124,7 +126,8 @@ def process_chunk(args):
     black_lens = array.array("i")
     stm = []
     target = []
-    material = []
+    score_probe_val = []
+    score_probe_valid = []
     n_total = len(positions)
     n_skipped = 0
 
@@ -145,10 +148,12 @@ def process_chunk(args):
 
         stm.append(1 if board.turn == chess.WHITE else 0)
         target.append(t)
-        material.append(material_balance(board))
+        s, valid = score_probe_target(pos)
+        score_probe_val.append(s)
+        score_probe_valid.append(valid)
 
     return (white_idx, white_lens, black_idx, black_lens, stm, target,
-            material, n_total, n_skipped)
+            score_probe_val, score_probe_valid, n_total, n_skipped)
 
 
 def main():
@@ -199,7 +204,8 @@ def main():
     black_off = [0]
     stm = []       # 1 = white to move, 0 = black
     target = []
-    material = []  # auxiliary material-probe target, mover-relative pawns
+    score_probe_val = []    # auxiliary score-probe target, clipped pawns
+    score_probe_valid = []  # False for exact/mate rows -- excluded from that loss
 
     n_total = 0
     n_used = 0
@@ -226,7 +232,9 @@ def main():
 
             stm.append(1 if board.turn == chess.WHITE else 0)
             target.append(t)
-            material.append(material_balance(board))
+            s, valid = score_probe_target(pos)
+            score_probe_val.append(s)
+            score_probe_valid.append(valid)
 
             n_used += 1
             if n_used % 200000 == 0:
@@ -244,7 +252,7 @@ def main():
                                             args.chunk_size))
         with Pool(processes=workers) as pool:
             for (w_idx, w_lens, b_idx, b_lens, chunk_stm, chunk_target,
-                 chunk_material, chunk_total,
+                 chunk_score_val, chunk_score_valid, chunk_total,
                  chunk_skipped) in pool.imap(process_chunk, chunks):
                 n_total += chunk_total
                 n_skipped += chunk_skipped
@@ -258,7 +266,8 @@ def main():
 
                 stm.extend(chunk_stm)
                 target.extend(chunk_target)
-                material.extend(chunk_material)
+                score_probe_val.extend(chunk_score_val)
+                score_probe_valid.extend(chunk_score_valid)
 
                 n_used += len(chunk_stm)
                 print(f"  ...{n_used} positions processed "
@@ -274,7 +283,10 @@ def main():
     np.save(outdir / "black_off.npy", np.array(black_off, dtype=np.int64))
     np.save(outdir / "stm.npy", np.array(stm, dtype=np.uint8))
     np.save(outdir / "target.npy", np.array(target, dtype=np.float32))
-    np.save(outdir / "material.npy", np.array(material, dtype=np.float32))
+    np.save(outdir / "score_probe_val.npy",
+            np.array(score_probe_val, dtype=np.float32))
+    np.save(outdir / "score_probe_valid.npy",
+            np.array(score_probe_valid, dtype=np.uint8))
 
     nnz_white = len(white_idx)
     nnz_black = len(black_idx)