summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorScott Gasch <[email protected]>2026-10-20 18:37:20 -0700
committerScott Gasch <[email protected]>2026-10-20 18:37:20 -0700
commit9cf14e14e6999420f234cda951894337e034f57c (patch)
treed4845d0cf127ef4226a5cd2e60472ee750213614
parentc8c6bbc2c1a103e5b07b079a0f8bdf66c7724d80 (diff)
NNUE: re-fit raw-to-score calibration, NNU4 weight format, super-lazy default-on, lazy probe measured net-negative
Fits an affine (a + b*raw) correction into every exported .nnue file (export_nnue_weights.py's CalibrateScoreMapping) so the engine's consumed centipawn output tracks Stockfish's real score scale rather than the net's nominal raw WDL-sigmoid output -- training variance alone shifts this mapping across retrains even with an identical recipe, so it's re-fit and baked in per export rather than assumed constant. Weight format bumped to "NNU4" to carry the two new floats; older NNU1/2/3 files are no longer readable. Super-lazy (material-only) pre-filter promoted to default-on (g_fUseNnueSuperLazy = TRUE) with a recalibrated per-material-bucket margin table (measured against the recalibrated net, 16-shard cross_sectional_200k.ep_ sample) -- real, measured nps win across all three curated suites. The separate lazy-probe gate (accumulator-based early exit) is measured net-negative on real search traffic at this net's probe-vs-forward latency ratio and stays off by default (g_fUseNnueLazyProbe = FALSE); code kept live since margin tuning could still flip this, per docs/NNUE.md. Also: direct per-call latency instrumentation for NnueProbeScore/ NnueForward (root.c, EVAL_TIME build) to make the lazy-probe breakeven math measurable directly instead of inferred; QSearch futility margins given real-data-derived NNUE_FULL/NNUE_LAZY starting points. Co-Authored-By: Claude Sonnet 5 <[email protected]>
-rw-r--r--annotate_pgns/build_training_tensors.py64
-rw-r--r--annotate_pgns/export_nnue_weights.py95
-rw-r--r--annotate_pgns/nnue_model.py84
-rw-r--r--annotate_pgns/nnue_train.py139
-rw-r--r--annotate_pgns/run_nnue_pipeline.py57
-rwxr-xr-xpgn/twic.pl6
-rwxr-xr-xsrc/chess.h25
-rw-r--r--src/command.c12
-rw-r--r--src/docs/NNUE.md407
-rw-r--r--src/eval.c129
-rw-r--r--src/nnueeval.c68
-rw-r--r--src/root.c53
12 files changed, 980 insertions, 159 deletions
diff --git a/annotate_pgns/build_training_tensors.py b/annotate_pgns/build_training_tensors.py
index 0f69d6c..ff9e772 100644
--- a/annotate_pgns/build_training_tensors.py
+++ b/annotate_pgns/build_training_tensors.py
@@ -51,16 +51,14 @@ import numpy as np
from nnue_feature_spec import active_feature_indices, chess
-# Standard piece values for the auxiliary material-probe target
-# (nnue_model.py's material_probe, added 2026-10-17 after a measured
-# failure -- see that module's docstring). Plain material count,
-# mover-relative, NOT run through the WDL sigmoid -- this is a
-# straight regression target, deliberately non-saturating. Kings
-# excluded (always 1-1 on a legal board, carries no signal).
-_PIECE_VALUES = {
- chess.PAWN: 1, chess.KNIGHT: 3, chess.BISHOP: 3,
- chess.ROOK: 5, chess.QUEEN: 9,
-}
+# Clip bound for the auxiliary score-probe target (nnue_model.py's
+# score_probe, added 2026-10-20 -- see that module's docstring). A
+# handful of blowout-but-not-forced-mate scores (see the user's own
+# +13.67-before-mate example) would otherwise dominate the probe's
+# MSE; exact/mate rows are excluded entirely (see SCORE_VALID below),
+# not just clipped, since a mate score isn't "the Stockfish score" in
+# the pawn sense this probe targets.
+SCORE_CLIP_PAWNS = 20.0
def position_target(pos, lam):
@@ -78,14 +76,18 @@ def position_target(pos, lam):
return lam * p_search + (1.0 - lam) * r_outcome
-def material_balance(board):
- """Mover-relative material balance in pawns, +ve = side to move
- is materially ahead."""
- total = 0
- for piece_type, value in _PIECE_VALUES.items():
- total += value * len(board.pieces(piece_type, chess.WHITE))
- total -= value * len(board.pieces(piece_type, chess.BLACK))
- return total if board.turn == chess.WHITE else -total
+def score_probe_target(pos):
+ """Returns (score, valid) for the auxiliary score-probe target --
+ Stockfish's own raw per-position score (pos["score"], already
+ mover-relative pawns per extract_selfplay_positions.py), clipped
+ to +-SCORE_CLIP_PAWNS, or (0.0, False) for exact/mate rows (no
+ "score" field, or not a real Stockfish-judged centipawn-ish value
+ -- same exclusion reasoning as p_search's, see NNUE.md)."""
+ if pos.get("exact") or "score" not in pos:
+ return 0.0, False
+ s = pos["score"]
+ s = max(-SCORE_CLIP_PAWNS, min(SCORE_CLIP_PAWNS, s))
+ return s, True
def iter_positions(path):
@@ -124,7 +126,8 @@ def process_chunk(args):
black_lens = array.array("i")
stm = []
target = []
- material = []
+ score_probe_val = []
+ score_probe_valid = []
n_total = len(positions)
n_skipped = 0
@@ -145,10 +148,12 @@ def process_chunk(args):
stm.append(1 if board.turn == chess.WHITE else 0)
target.append(t)
- material.append(material_balance(board))
+ s, valid = score_probe_target(pos)
+ score_probe_val.append(s)
+ score_probe_valid.append(valid)
return (white_idx, white_lens, black_idx, black_lens, stm, target,
- material, n_total, n_skipped)
+ score_probe_val, score_probe_valid, n_total, n_skipped)
def main():
@@ -199,7 +204,8 @@ def main():
black_off = [0]
stm = [] # 1 = white to move, 0 = black
target = []
- material = [] # auxiliary material-probe target, mover-relative pawns
+ score_probe_val = [] # auxiliary score-probe target, clipped pawns
+ score_probe_valid = [] # False for exact/mate rows -- excluded from that loss
n_total = 0
n_used = 0
@@ -226,7 +232,9 @@ def main():
stm.append(1 if board.turn == chess.WHITE else 0)
target.append(t)
- material.append(material_balance(board))
+ s, valid = score_probe_target(pos)
+ score_probe_val.append(s)
+ score_probe_valid.append(valid)
n_used += 1
if n_used % 200000 == 0:
@@ -244,7 +252,7 @@ def main():
args.chunk_size))
with Pool(processes=workers) as pool:
for (w_idx, w_lens, b_idx, b_lens, chunk_stm, chunk_target,
- chunk_material, chunk_total,
+ chunk_score_val, chunk_score_valid, chunk_total,
chunk_skipped) in pool.imap(process_chunk, chunks):
n_total += chunk_total
n_skipped += chunk_skipped
@@ -258,7 +266,8 @@ def main():
stm.extend(chunk_stm)
target.extend(chunk_target)
- material.extend(chunk_material)
+ score_probe_val.extend(chunk_score_val)
+ score_probe_valid.extend(chunk_score_valid)
n_used += len(chunk_stm)
print(f" ...{n_used} positions processed "
@@ -274,7 +283,10 @@ def main():
np.save(outdir / "black_off.npy", np.array(black_off, dtype=np.int64))
np.save(outdir / "stm.npy", np.array(stm, dtype=np.uint8))
np.save(outdir / "target.npy", np.array(target, dtype=np.float32))
- np.save(outdir / "material.npy", np.array(material, dtype=np.float32))
+ np.save(outdir / "score_probe_val.npy",
+ np.array(score_probe_val, dtype=np.float32))
+ np.save(outdir / "score_probe_valid.npy",
+ np.array(score_probe_valid, dtype=np.uint8))
nnz_white = len(white_idx)
nnz_black = len(black_idx)
diff --git a/annotate_pgns/export_nnue_weights.py b/annotate_pgns/export_nnue_weights.py
index 4c8cb99..d26ab93 100644
--- a/annotate_pgns/export_nnue_weights.py
+++ b/annotate_pgns/export_nnue_weights.py
@@ -4,8 +4,8 @@ the C engine (src/nnueeval.c's NnueLoadWeights) can read directly --
no parsing, just a fixed-order sequence of arrays behind a small
header.
-File format (all little-endian), version "NNU3":
- magic: 4 bytes, ASCII "NNU3"
+File format (all little-endian), version "NNU4":
+ magic: 4 bytes, ASCII "NNU4"
dims: 4x uint32 -- NNUE_DIMENSIONS, L1, L2, L3 (sanity-checked
against the C side's own compile-time constants on
load, so a mismatched build can't silently misread the
@@ -24,6 +24,15 @@ File format (all little-endian), version "NNU3":
CalibrateActivationScales) with headroom, NOT from a
theoretical max -- see docs/NNUE.md's "Quantization
plan" / the int8 float-vs-quantized correctness check.
+ calib: 2x float32 (CALIB_A, CALIB_B) -- affine correction so
+ the engine's consumed centipawn output tracks
+ Stockfish's own real score, not the net's nominal raw
+ WDL-sigmoid scale (which drifts run-to-run even with
+ an identical K/lambda/loss-exponent recipe -- see
+ CalibrateScoreMapping and the 2026-10-20 "keeping this
+ stable across retrains" conversation). NnueForward
+ applies `(CALIB_A + CALIB_B*fOut) * 100.0f` instead of
+ a bare `fOut * 100.0f`.
w_scale: 3x float32 (WS1, WS2, WS3) -- fixed-point scales for
fc1.weight/fc2.weight/fc3.weight (int8), one per layer,
each derived from that layer's own max|.| (exact, not
@@ -56,7 +65,8 @@ Usage:
python3 export_nnue_weights.py \
runs/sd10_twic_2M/nnue_ckpt/ckpt_epoch019.pt \
runs/sd10_twic_2M/nnue_ckpt/weights.nnue \
- --tensor-dir runs/sd10_twic_2M/tensors_train
+ --tensor-dir runs/sd10_twic_2M/tensors_train \
+ --holdout-tensor-dir runs/sd10_twic_2M/tensors_holdout
"""
import argparse
import struct
@@ -136,6 +146,74 @@ def CalibrateActivationScales(model, tensor_dir, n_sample=50000, seed=42):
return tuple(scales)
+def CalibrateScoreMapping(model, holdout_tensor_dir, n_sample=20000, seed=7):
+ """Fits a simple affine correction (a, b) so that
+ a + b*raw tracks Stockfish's own real score (pawns, mover-
+ relative) -- not raw's nominal WDL-sigmoid scale. See the
+ 2026-10-20 "keeping this stable across retrains" conversation:
+ a fixed K/lambda/loss-exponent recipe is NOT enough on its own to
+ guarantee the engine's consumed cp output stays on a stable scale
+ across generations -- training variance (data, convergence) still
+ shifts the empirical raw-to-score mapping even with an identical
+ recipe. Re-fitting (a, b) every generation and baking the
+ correction into the exported format, applied in NnueForward
+ itself, decouples engine search-tuning from needing the raw net's
+ internal scale to stay perfectly constant forever.
+
+ Fit against the HOLDOUT tensor dir specifically (not train) --
+ an honest estimate needs positions the checkpoint wasn't fit on.
+ Uses score_probe_val.npy/score_probe_valid.npy (Stockfish's own
+ clipped per-position score, and a validity mask excluding exact/
+ mate rows -- see build_training_tensors.py), which
+ build_training_tensors.py already writes for every tensor dir
+ regardless of whether --score-loss-weight was ever used during
+ training. Ordinary least squares, free intercept (not forced
+ through the origin)."""
+ d = Path(holdout_tensor_dir)
+ white_idx = np.load(d / "white_idx.npy"); white_off = np.load(d / "white_off.npy")
+ black_idx = np.load(d / "black_idx.npy"); black_off = np.load(d / "black_off.npy")
+ stm = np.load(d / "stm.npy")
+ score_val = np.load(d / "score_probe_val.npy")
+ score_valid = np.load(d / "score_probe_valid.npy").astype(bool)
+
+ valid_rows = np.flatnonzero(score_valid)
+ if len(valid_rows) == 0:
+ sys.exit(f"CalibrateScoreMapping: {d} has no valid score-probe rows "
+ f"(all exact/mate?) -- can't fit a calibration against it")
+
+ rng = np.random.default_rng(seed)
+ rows = rng.choice(valid_rows, size=min(n_sample, len(valid_rows)), replace=False)
+ rows.sort()
+
+ raws = []
+ with torch.no_grad():
+ for i in range(0, len(rows), 5000):
+ chunk = rows[i:i + 5000]
+ w_idx_list, w_off_list, b_idx_list, b_off_list = [], [], [], []
+ off_w, off_b = 0, 0
+ for r in chunk:
+ wi = white_idx[white_off[r]:white_off[r + 1]]
+ bi = black_idx[black_off[r]:black_off[r + 1]]
+ w_idx_list.append(wi); w_off_list.append(off_w); off_w += len(wi)
+ b_idx_list.append(bi); b_off_list.append(off_b); off_b += len(bi)
+ w_idx = torch.from_numpy(np.concatenate(w_idx_list).astype(np.int64))
+ w_off = torch.tensor(w_off_list, dtype=torch.int64)
+ b_idx = torch.from_numpy(np.concatenate(b_idx_list).astype(np.int64))
+ b_off = torch.tensor(b_off_list, dtype=torch.int64)
+ stm_t = torch.from_numpy(stm[chunk].astype(np.int64))
+ raws.append(model(w_idx, w_off, b_idx, b_off, stm_t).numpy())
+ raw = np.concatenate(raws).astype(np.float64)
+ score = score_val[rows].astype(np.float64)
+
+ # score ~= a + b*raw, ordinary least squares, free intercept.
+ b, a = np.polyfit(raw, score, 1)
+ resid = score - (a + b * raw)
+ print(f"CalibrateScoreMapping: fit on {len(rows)} holdout positions -- "
+ f"a={a:.4f}, b={b:.4f}, mean|resid|={np.mean(np.abs(resid)):.3f} pawns, "
+ f"resid p98={np.percentile(np.abs(resid), 98):.3f} pawns")
+ return float(a), float(b)
+
+
def QuantizeFt(arr):
"""Rounds + clamps a float32 array to int16 at FT_SCALE. Returns
(int16 ndarray, number of values that clipped)."""
@@ -241,6 +319,13 @@ def main():
help="a build_training_tensors.py output dir, used "
"to calibrate the FC head's activation scales "
"against real positions (e.g. tensors_train)")
+ ap.add_argument("--holdout-tensor-dir", required=True,
+ help="a build_training_tensors.py HOLDOUT output dir "
+ "(not the same as --tensor-dir), used to fit the "
+ "raw-to-real-Stockfish-score affine correction "
+ "(a, b) baked into the exported format -- see "
+ "CalibrateScoreMapping. Must be holdout, not "
+ "train, for an honest (non-overfit) fit.")
args = ap.parse_args()
ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=True)
@@ -276,6 +361,7 @@ def main():
model.eval()
AS1, AS2, AS3 = CalibrateActivationScales(model, args.tensor_dir)
+ CALIB_A, CALIB_B = CalibrateScoreMapping(model, args.holdout_tensor_dir)
ft_weight_f32 = sd["ft.weight"].detach().contiguous().to(torch.float32).numpy()
ft_bias_f32 = sd["ft_bias"].detach().contiguous().to(torch.float32).numpy()
@@ -302,11 +388,12 @@ def main():
f"(see docs/NNUE.md)")
with open(args.outfile, "wb") as f:
- f.write(b"NNU3")
+ f.write(b"NNU4")
f.write(struct.pack("<4I", DIMENSIONS, L1, L2, L3))
f.write(struct.pack("<f", FT_SCALE))
f.write(struct.pack("<3f", AS1, AS2, AS3))
f.write(struct.pack("<3f", WS1, WS2, WS3))
+ f.write(struct.pack("<2f", CALIB_A, CALIB_B))
f.write(ft_weight_q.tobytes())
f.write(ft_bias_q.tobytes())
f.write(fc1_w_q.tobytes())
diff --git a/annotate_pgns/nnue_model.py b/annotate_pgns/nnue_model.py
index c483ce7..47dca76 100644
--- a/annotate_pgns/nnue_model.py
+++ b/annotate_pgns/nnue_model.py
@@ -19,18 +19,58 @@ prefilter. All irrelevant to proving the training pipeline works.
2026-10-17: added `material_probe`, a small auxiliary linear head on
the same concatenated accumulator `fc1` reads, trained jointly with
-the main WDL loss (see nnue_train.py's --material-loss-weight) against
-each position's raw material balance (mover-relative pawns, no
-sigmoid). Added after a real, measured failure: a net trained with
-plain MSE (and separately, exponent=2.6) against WDL-sigmoid targets
-alone could not tell a lost pawn from a lost queen in crowded midgame
-positions (see sanity_check_net.py) -- directionally correct but
-magnitude-blind. The probe supplies a non-saturating gradient straight
-at material, forcing `ft`'s embedding weights to keep material
-linearly recoverable even where the main WDL loss has little pressure
-to care. It's training-only scaffolding, like the SF doc's "virtual
-features" factorization trick -- dropped before export, zero
-inference-time or quantization-format cost.
+the main WDL loss against each position's raw material balance
+(mover-relative pawns, no sigmoid). Added after a real, measured
+failure: a net trained with plain MSE (and separately, exponent=2.6)
+against WDL-sigmoid targets alone could not tell a lost pawn from a
+lost queen in crowded midgame positions (see sanity_check_net.py) --
+directionally correct but magnitude-blind.
+
+2026-10-20: replaced by `score_probe` (same mechanism, different
+target -- see that section below) and removed. Once `score_probe`
+regresses Stockfish's actual raw score directly, `material_probe`'s
+target (piece count) is a strict special case of what `score_probe`
+already has to recover -- material dominates Stockfish's own score in
+almost any position, so a net that fits the real score well has
+implicitly recovered material too. Kept as one objective instead of
+two overlapping ones; re-add material_probe alongside score_probe only
+if a future sanity check shows score_probe alone is still
+material-blind in some case it doesn't already cover.
+
+2026-10-20: tried `score_probe`, same mechanism as material_probe
+(small Linear head on the same accumulator, trained jointly, dropped
+before export) targeting Stockfish's own raw per-position score
+directly via plain MSE -- motivated by a measured failure of a
+different kind from material_probe's: in clearly-losing positions,
+the main WDL head's score magnitude did not reflect how lost the
+position actually was (-2 for "lost" and -3 for "about to be mated",
+no meaningful gradation), because the WDL loss only ever supervises
+`raw` through `sigmoid(raw/K)` against a win-probability target, which
+saturates hard once a position is clearly decided (K=2.0659 on the
+2026-10-20 corpus -- sigmoid(8/K)=0.9796 vs sigmoid(13/K)=0.9982, a
+target spread of under 0.02 for a 5-pawn difference in the real
+score). **Measured, then reverted**: a medium-scale (500k positions,
+~15 epochs) controlled comparison (score_probe-weighted run vs. an
+otherwise-identical control) showed only a small improvement in how
+well `forward()`'s own raw output tracked the real score at the
+losing extremes -- compression was still clearly present in both.
+Root cause: score_probe's gradient only ever reaches the *shared
+accumulator* (`ft`/`ft_p`/`ft_bias`), never `fc1`/`fc2`/`fc3` --
+nothing forces the actually-exported path to widen its own range, it
+can only optionally make use of a richer upstream representation if
+doing so happens to help the (still-saturating) WDL loss too.
+
+2026-10-20: replaced score_probe with a direct, unsaturated regression
+term on `raw` itself: `MSE(raw, clipped_score)`, added straight
+alongside the WDL loss in nnue_train.py, no auxiliary head at all.
+This puts gradient directly on `fc1`/`fc2`/`fc3` -- the actual
+exported path -- rather than hoping a richer accumulator happens to
+propagate through. Not a competing objective with the WDL loss, either:
+since `p_search = sigmoid(score/K)`, the WDL loss's own implied ideal
+`raw` is already approximately `score` (especially at lambda close to
+1.0) -- this term is an unsaturated restatement of the same target,
+not a different one, so no separate probe/model-surface-area is
+needed to get its benefit. See nnue_train.py's --score-loss-weight.
2026-10-18: added `ft_p`, the actual Stockfish-doc "P" factor (see
NNUE.md's "feature factorization" status entry) -- a second,
@@ -96,15 +136,9 @@ class NnueNet(nn.Module):
self.fc2 = nn.Linear(l2, l3)
self.fc3 = nn.Linear(l3, 1)
- # Auxiliary, training-only head -- see module docstring's
- # 2026-10-17 note. Reads the same concatenated accumulator
- # fc1 does; not used at inference/export time.
- self.material_probe = nn.Linear(2 * l1, 1)
-
def accumulator(self, white_idx, white_off, black_idx, black_off, stm):
"""The ReLU'd, own-perspective-first concatenated accumulator
- (shape [B, 2*l1]) -- shared input to both fc1 (the real eval
- head) and material_probe (the auxiliary head). ft_p's
+ (shape [B, 2*l1]) -- input to fc1. ft_p's
contribution uses the identical bag offsets as ft -- same
positions, same active-feature counts, just a second lookup
of each index modulo ps_nb into the smaller, denser table."""
@@ -134,18 +168,6 @@ class NnueNet(nn.Module):
x = self.fc3(x)
return x.view(-1)
- def forward_with_material(self, white_idx, white_off, black_idx, black_off, stm):
- """Like forward(), but also returns the auxiliary material
- probe's prediction (mover-relative pawns, no sigmoid) off the
- same accumulator -- one extra Linear(2*l1, 1), training-only.
- Returns (raw_score, material_pred), both shape [B]."""
- x = self.accumulator(white_idx, white_off, black_idx, black_off, stm)
- material_pred = self.material_probe(x).view(-1)
- y = torch.relu(self.fc1(x))
- y = torch.relu(self.fc2(y))
- y = self.fc3(y)
- return y.view(-1), material_pred
-
def win_probability(self, white_idx, white_off, black_idx, black_off, stm, k=DEFAULT_K):
raw = self.forward(white_idx, white_off, black_idx, black_off, stm)
return torch.sigmoid(raw / k)
diff --git a/annotate_pgns/nnue_train.py b/annotate_pgns/nnue_train.py
index ea3d0a9..e413aa9 100644
--- a/annotate_pgns/nnue_train.py
+++ b/annotate_pgns/nnue_train.py
@@ -42,17 +42,22 @@ class CsrTensors:
self.black_off = np.load(tensor_dir / "black_off.npy")
self.stm = np.load(tensor_dir / "stm.npy")
self.target = np.load(tensor_dir / "target.npy")
- # material.npy added 2026-10-17 (see nnue_model.py's
- # material_probe) -- optional for backward compat with
- # tensor dirs built before this field existed.
- material_path = tensor_dir / "material.npy"
- self.material = (np.load(material_path) if material_path.exists()
- else np.zeros_like(self.target))
+ # score_probe_val.npy/score_probe_valid.npy added 2026-10-20
+ # (see nnue_model.py's score_probe, replacing material_probe)
+ # -- optional for backward compat with tensor dirs built before
+ # this field existed.
+ score_path = tensor_dir / "score_probe_val.npy"
+ valid_path = tensor_dir / "score_probe_valid.npy"
+ self.score_probe_val = (np.load(score_path) if score_path.exists()
+ else np.zeros_like(self.target))
+ self.score_probe_valid = (np.load(valid_path) if valid_path.exists()
+ else np.zeros_like(self.target, dtype=np.uint8))
self.n = len(self.target)
assert len(self.white_off) == self.n + 1
assert len(self.black_off) == self.n + 1
assert len(self.stm) == self.n
- assert len(self.material) == self.n
+ assert len(self.score_probe_val) == self.n
+ assert len(self.score_probe_valid) == self.n
def __len__(self):
return self.n
@@ -93,7 +98,8 @@ def iterate_batches(data: CsrTensors, batch_size, shuffle, rng):
b_idx, b_off = _gather_batch(data.black_idx, data.black_off, rows_sorted)
stm = data.stm[rows_sorted]
target = data.target[rows_sorted]
- material = data.material[rows_sorted]
+ score_probe_val = data.score_probe_val[rows_sorted]
+ score_probe_valid = data.score_probe_valid[rows_sorted]
yield (
torch.from_numpy(w_idx.astype(np.int64, copy=False)),
torch.from_numpy(w_off),
@@ -101,7 +107,8 @@ def iterate_batches(data: CsrTensors, batch_size, shuffle, rng):
torch.from_numpy(b_off),
torch.from_numpy(stm.astype(np.int64, copy=False)),
torch.from_numpy(target.copy()),
- torch.from_numpy(material.astype(np.float32, copy=True)),
+ torch.from_numpy(score_probe_val.astype(np.float32, copy=True)),
+ torch.from_numpy(score_probe_valid.astype(np.float32, copy=True)),
)
@@ -118,34 +125,63 @@ def powered_loss(pred, target, exponent, reduction="mean"):
return err.sum() if reduction == "sum" else err.mean()
-def evaluate(model, data, batch_size, k, loss_exponent, device="cpu"):
- """Returns (wdl_loss, material_mse) -- kept separate rather than
- pre-combined so the log/checkpoint can show whether the material
- probe is actually converging, independently of whatever weight
+def score_loss_fn(raw, score_val, score_valid, score_loss_type, huber_delta):
+ if score_loss_type == "huber":
+ return masked_huber_sum(raw, score_val, score_valid, huber_delta)
+ return masked_mse_sum(raw, score_val, score_valid)
+
+
+def evaluate(model, data, batch_size, k, loss_exponent, device="cpu",
+ score_loss_type="mse", huber_delta=1.0):
+ """Returns (wdl_loss, score_mse) -- kept separate rather than
+ pre-combined so the log/checkpoint can show whether `raw` is
+ actually tracking the real score, independently of whatever weight
the caller gives it in the training loss."""
model.eval()
total_wdl_loss = 0.0
- total_material_loss = 0.0
+ total_score_loss = 0.0
total_n = 0
+ total_score_valid = 0
rng = np.random.default_rng(0)
with torch.no_grad():
- for w_idx, w_off, b_idx, b_off, stm, target, material in iterate_batches(
+ for w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid in iterate_batches(
data, batch_size, shuffle=False, rng=rng):
- w_idx, w_off, b_idx, b_off, stm, target, material = _to_device(
- (w_idx, w_off, b_idx, b_off, stm, target, material), device)
- raw, material_pred = model.forward_with_material(w_idx, w_off, b_idx, b_off, stm)
+ w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid = _to_device(
+ (w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid), device)
+ raw = model.forward(w_idx, w_off, b_idx, b_off, stm)
pred = torch.sigmoid(raw / k)
wdl_loss = powered_loss(pred, target, loss_exponent, reduction="sum")
- material_loss = nn_mse_sum(material_pred, material)
+ score_loss = score_loss_fn(raw, score_val, score_valid, score_loss_type, huber_delta)
total_wdl_loss += wdl_loss.item()
- total_material_loss += material_loss.item()
+ total_score_loss += score_loss.item()
total_n += len(target)
+ total_score_valid += int(score_valid.sum().item())
model.train()
- return total_wdl_loss / max(total_n, 1), total_material_loss / max(total_n, 1)
+ return (total_wdl_loss / max(total_n, 1),
+ total_score_loss / max(total_score_valid, 1))
+
+
+def masked_mse_sum(pred, target, valid):
+ """Sum of squared error over only the rows where valid != 0 --
+ exact/mate rows carry no real score target (see
+ build_training_tensors.py's score_probe_target) and must not pull
+ `raw` toward an arbitrary 0.0 placeholder."""
+ return ((pred - target).pow(2) * valid).sum()
-def nn_mse_sum(pred, target):
- return (pred - target).pow(2).sum()
+def masked_huber_sum(pred, target, valid, delta):
+ """Sum of Huber loss over only the rows where valid != 0 (same
+ masking as masked_mse_sum). Quadratic for |error| < delta
+ (preserves fine resolution on close, contested positions -- the
+ ones that actually matter for move choice), linear beyond it
+ (bounds how much a handful of large-magnitude blowout positions
+ can dominate the loss/gradient, unlike plain MSE). delta is in the
+ same pawn units as `pred`/`target`."""
+ err = (pred - target).abs()
+ quad = 0.5 * err.pow(2)
+ lin = delta * (err - 0.5 * delta)
+ loss = torch.where(err < delta, quad, lin)
+ return (loss * valid).sum()
def main():
@@ -164,16 +200,26 @@ def main():
"loss ratio between small and large WDL-space "
"errors, pushing harder to fix decisively-wrong "
"predictions. See powered_loss()'s docstring.")
- ap.add_argument("--material-loss-weight", type=float, default=0.0,
- help="weight on the auxiliary material_probe MSE "
- "term (plain pawns^2, NOT WDL-space -- so this "
- "is typically orders of magnitude larger in raw "
+ ap.add_argument("--score-loss-type", choices=["mse", "huber"], default="mse",
+ help="loss shape for the direct raw-vs-score term. "
+ "'mse': plain squared error (simple, but a "
+ "handful of large-magnitude blowout positions "
+ "can dominate it). 'huber': quadratic below "
+ "--huber-delta, linear beyond it -- bounds "
+ "blowout influence while keeping fine "
+ "resolution on close/contested positions.")
+ ap.add_argument("--huber-delta", type=float, default=1.0,
+ help="Huber loss transition point, in pawns -- "
+ "only used when --score-loss-type huber.")
+ ap.add_argument("--score-loss-weight", type=float, default=0.0,
+ help="weight on a direct MSE(raw, clipped Stockfish "
+ "score) term, straight on fc3's own output "
+ "(plain pawns^2, NOT WDL-space -- so this is "
+ "typically orders of magnitude larger in raw "
"scale than the WDL loss; start small, e.g. "
- "0.001-0.01, and check sanity_check_net.py "
- "rather than assume a value works). 0.0 "
- "(default) disables it entirely -- the probe "
- "head still exists but gets no gradient, "
- "equivalent to not having added it.")
+ "0.001, and check both holdout losses in "
+ "train_log.jsonl rather than assume a value "
+ "works). 0.0 (default) disables it entirely.")
ap.add_argument("--threads", type=int, default=16,
help="torch CPU thread count -- kept below this box's "
"24 cores (not maxed out) since this is a shared "
@@ -277,16 +323,17 @@ def main():
epoch_n = 0
t_epoch = time.time()
- for w_idx, w_off, b_idx, b_off, stm, target, material in iterate_batches(
+ for w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid in iterate_batches(
train_data, args.batch_size, shuffle=True, rng=rng):
- w_idx, w_off, b_idx, b_off, stm, target, material = _to_device(
- (w_idx, w_off, b_idx, b_off, stm, target, material), args.device)
+ w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid = _to_device(
+ (w_idx, w_off, b_idx, b_off, stm, target, score_val, score_valid), args.device)
optimizer.zero_grad()
- raw, material_pred = model.forward_with_material(w_idx, w_off, b_idx, b_off, stm)
+ raw = model.forward(w_idx, w_off, b_idx, b_off, stm)
pred = torch.sigmoid(raw / args.k)
wdl_loss = powered_loss(pred, target, args.loss_exponent)
- material_loss = nn_mse_sum(material_pred, material) / len(target)
- loss = wdl_loss + args.material_loss_weight * material_loss
+ score_loss = score_loss_fn(raw, score_val, score_valid,
+ args.score_loss_type, args.huber_delta) / len(target)
+ loss = wdl_loss + args.score_loss_weight * score_loss
loss.backward()
optimizer.step()
@@ -296,34 +343,36 @@ def main():
global_step += 1
if global_step % args.eval_every_steps == 0:
- holdout_wdl, holdout_material = evaluate(
+ holdout_wdl, holdout_score = evaluate(
model, holdout_data, args.batch_size, args.k,
- args.loss_exponent, args.device)
+ args.loss_exponent, args.device,
+ args.score_loss_type, args.huber_delta)
elapsed = time.time() - t_start
rec = {
"step": global_step, "epoch": epoch,
"train_mse_running": epoch_loss / max(epoch_n, 1),
"holdout_mse": holdout_wdl,
- "holdout_material_mse": holdout_material,
+ "holdout_score_probe_mse": holdout_score,
"elapsed_s": round(elapsed, 1),
"positions_seen": epoch_n + epoch * len(train_data),
}
print(f" step {global_step} epoch {epoch} "
f"train_mse={rec['train_mse_running']:.5f} "
f"holdout_mse={holdout_wdl:.5f} "
- f"holdout_material_mse={holdout_material:.4f} "
+ f"holdout_score_probe_mse={holdout_score:.4f} "
f"elapsed={elapsed:.0f}s")
log_f.write(json.dumps(rec) + "\n")
log_f.flush()
- holdout_wdl, holdout_material = evaluate(
+ holdout_wdl, holdout_score = evaluate(
model, holdout_data, args.batch_size, args.k,
- args.loss_exponent, args.device)
+ args.loss_exponent, args.device,
+ args.score_loss_type, args.huber_delta)
holdout_mse = holdout_wdl
epoch_elapsed = time.time() - t_epoch
print(f"Epoch {epoch} done: train_mse={epoch_loss / max(epoch_n, 1):.5f} "
f"holdout_mse={holdout_wdl:.5f} "
- f"holdout_material_mse={holdout_material:.4f} ({epoch_elapsed:.0f}s)")
+ f"holdout_score_probe_mse={holdout_score:.4f} ({epoch_elapsed:.0f}s)")
ckpt_path = outdir / f"ckpt_epoch{epoch:03d}.pt"
torch.save({
diff --git a/annotate_pgns/run_nnue_pipeline.py b/annotate_pgns/run_nnue_pipeline.py
index 39bc487..6e51416 100644
--- a/annotate_pgns/run_nnue_pipeline.py
+++ b/annotate_pgns/run_nnue_pipeline.py
@@ -199,6 +199,21 @@ def main():
help="build_training_tensors.py's "
"target = lambda*p_search + (1-lambda)*r_outcome "
"for non-exact rows")
+ ap.add_argument("--k", type=float, default=None,
+ help="nnue_train.py's --k (the WDL-sigmoid scale, "
+ "pred=sigmoid(raw/k)) -- previously NOT passed "
+ "through at all, silently falling back to "
+ "nnue_train.py's own DEFAULT_K (1.3123, a Phase-A "
+ "placeholder), regardless of what K "
+ "fit_k_and_label.py actually fit for this corpus's "
+ "own p_search labels (stage 3 below) -- a real "
+ "label/loss K mismatch, fixed 2026-10-20. Required "
+ "now (no default) specifically so this can't go "
+ "silently unmatched again: read fit_k_and_label.py's "
+ "own printed 'Fitted K = ...' line for this corpus "
+ "and pass that (times whatever saturation-"
+ "correction factor empirical testing supports -- "
+ "see docs/NNUE.md's K-recalibration status).")
ap.add_argument("--epochs", type=int, default=20)
ap.add_argument("--patience", type=int, default=0,
help="nnue_train.py's --patience (0 disables early "
@@ -206,12 +221,21 @@ def main():
ap.add_argument("--batch-size", type=int, default=8192)
ap.add_argument("--lr", type=float, default=1e-3)
ap.add_argument("--loss-exponent", type=float, default=2.0)
- ap.add_argument("--material-loss-weight", type=float, default=0.0)
+ ap.add_argument("--score-loss-weight", type=float, default=0.0)
ap.add_argument("--force", action="store_true",
help="re-run every stage even if its outputs "
"already exist")
args = ap.parse_args()
+ if args.k is None:
+ sys.exit("--k is required (no default, deliberately -- see its "
+ "help text): pass the K you actually want nnue_train.py's "
+ "loss to use. Do not guess -- run fit_k_and_label.py "
+ "first (or check its log from stage 3 below) to see what "
+ "it fits for THIS corpus, and apply whatever saturation-"
+ "correction factor has been validated empirically before "
+ "trusting its raw fit.")
+
pgn_paths = sorted({p for pattern in args.pgn_glob
for p in glob.glob(pattern)})
if not pgn_paths:
@@ -361,8 +385,8 @@ def main():
"--holdout-dir", tensors_holdout_dir,
"--outdir", ckpt_dir, *resume_flag,
"--epochs", args.epochs, "--batch-size", args.batch_size,
- "--lr", args.lr, "--loss-exponent", args.loss_exponent,
- "--material-loss-weight", args.material_loss_weight,
+ "--lr", args.lr, "--k", args.k, "--loss-exponent", args.loss_exponent,
+ "--score-loss-weight", args.score_loss_weight,
"--threads", args.threads, "--device", args.device,
"--patience", args.patience],
args.force)
@@ -372,10 +396,33 @@ def main():
run_stage(
"export_nnue_weights.py", [args.outfile],
[sys.executable, HERE / "export_nnue_weights.py",
- ckpt, args.outfile, "--tensor-dir", tensors_train_dir],
+ ckpt, args.outfile, "--tensor-dir", tensors_train_dir,
+ "--holdout-tensor-dir", tensors_holdout_dir],
+ args.force)
+
+ # 8. Refit the lazy-eval probe (nnue_vs_head.sh's NNUE_PROBE_WEIGHTS /
+ # command.c's "nnue probeload") against THIS checkpoint -- it's a
+ # separate offline ridge regression, not part of the exported .nnue
+ # file, and must be re-fit every generation (same rule as the
+ # calibration inside export_nnue_weights.py: not portable across
+ # checkpoints). Written alongside outfile, not into it. See
+ # fit_lazy_probe.py and the 2026-10-20 "keeping this stable across
+ # retrains" / lazy-eval conversation -- this is the regression fit
+ # only; the real search-node-traffic margin-safety calibration
+ # (CALIBRATE_MARGIN_SAFETY) is a separate, not-yet-automated step,
+ # deliberately not run here every generation (see that conversation
+ # for why it may not need to be).
+ probe_outfile = Path(args.outfile).with_name(
+ Path(args.outfile).stem + "_probe.bin")
+ run_stage(
+ "fit_lazy_probe.py", [probe_outfile],
+ [sys.executable, HERE / "fit_lazy_probe.py",
+ ckpt, probe_outfile,
+ "--train-tensor-dir", tensors_train_dir,
+ "--holdout-tensor-dir", tensors_holdout_dir],
args.force)
- print(f"\nPipeline complete: {args.outfile}")
+ print(f"\nPipeline complete: {args.outfile} (lazy probe: {probe_outfile})")
if __name__ == "__main__":
diff --git a/pgn/twic.pl b/pgn/twic.pl
deleted file mode 100755
index 1fccae2..0000000
--- a/pgn/twic.pl
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/perl
-
-for ($x = 608; $x > 0; $x--)
-{
- system "fetch http://www.chesscenter.com/twic/zips/twic" . $x . "g.zip";
-}
diff --git a/src/chess.h b/src/chess.h
index 3e65e7f..34099b2 100755
--- a/src/chess.h
+++ b/src/chess.h
@@ -1431,6 +1431,25 @@ typedef struct _COUNTERS
UINT64 u64CyclesSuperLazyExit;
UINT64 u64CyclesLazyExit;
UINT64 u64CyclesFullEvalExit;
+
+ // 2026-10-20: direct per-call latency of the two NNUE eval
+ // primitives (nnueeval.c), as opposed to the HCE-oriented
+ // whole-path-cost counters above (which NNUE's Eval() branch
+ // doesn't feed -- it's gated on g_fUseNnueEval at runtime, not
+ // #ifdef LAZY_EVAL). u64CyclesNnueProbeCall/u64CyclesNnueForwardCall
+ // time just the TIMED_EVAL_CALL-wrapped NnueProbeScore()/
+ // NnueForward() calls themselves (eval.c's production NNUE
+ // branch); u64NnueProbeAttempts counts every probe call made
+ // (hit or miss) -- unlike u64LazyEvals, which only counts
+ // hits -- needed as the denominator for a true avg-cost-per-
+ // attempt (the question that actually matters for judging
+ // whether the gate pays for itself: cost is paid on every
+ // attempt, the saving only on a hit). u64FullEvals already
+ // counts NnueForward calls 1:1 on this path, so no separate
+ // attempt counter is needed for it.
+ UINT64 u64CyclesNnueProbeCall;
+ UINT64 u64CyclesNnueForwardCall;
+ UINT64 u64NnueProbeAttempts;
}
tree;
@@ -2378,6 +2397,12 @@ extern CHAR g_rgfNnueFc3Weight[NNUE_L3];
extern float g_rgfNnueFc3Bias;
extern float g_rgfNnueActScale[3];
extern float g_rgfNnueWeightScale[3];
+// Affine raw-to-real-score correction (a + b*raw), re-fit and baked
+// into every export by CalibrateScoreMapping -- see nnueeval.c's
+// declaration comment and the 2026-10-20 "keeping this stable across
+// retrains" conversation. Defaults (0,1) are the identity transform.
+extern float g_fNnueCalibA;
+extern float g_fNnueCalibB;
extern FLAG g_fNnueWeightsLoaded;
extern FLAG g_fUseNnueEval;
diff --git a/src/command.c b/src/command.c
index 1767e98..9808c33 100644
--- a/src/command.c
+++ b/src/command.c
@@ -992,14 +992,22 @@ Return value:
if (!STRCMPI(argv[1], "superlazyon"))
{
+ // 2026-10-20: margin cp is now optional -- omit it to use
+ // eval.c's calibrated iNnueSuperLazyMarginByMatBucket table
+ // (the normal case); pass one to force a flat override across
+ // every material bucket instead, same convention as
+ // "nnue lazymargin".
if (argc < 3)
{
- Trace("Usage: nnue superlazyon <margin cp>\n");
+ g_iNnueSuperLazyMargin = -1;
+ g_fUseNnueSuperLazy = TRUE;
+ Trace("NNUE super-lazy (material-only) pre-filter enabled, "
+ "using calibrated per-bucket margin table.\n");
return;
}
g_iNnueSuperLazyMargin = (SCORE)atoi(argv[2]);
g_fUseNnueSuperLazy = TRUE;
- Trace("NNUE super-lazy (material-only) pre-filter enabled, margin=%d cp.\n",
+ Trace("NNUE super-lazy (material-only) pre-filter enabled, flat margin=%d cp.\n",
g_iNnueSuperLazyMargin);
return;
}
diff --git a/src/docs/NNUE.md b/src/docs/NNUE.md
index dbc5d54..d1481c6 100644
--- a/src/docs/NNUE.md
+++ b/src/docs/NNUE.md
@@ -2690,3 +2690,410 @@ adding a new orchestrator script.
architecture or export-format change -- so there's no reason to
gate it behind the factorization/augmentation experiments above;
can be added and swept independently.
+
+## Status (2026-10-20): losing-position score compression fixed (K
+## recalibration), per-generation score calibration automated, lazy
+## probe refit wired into the pipeline, margin-safety recalibrated
+## against real search traffic, LOP-based promotion gate added
+
+Prompted by a direct observation: the production net's score in
+clearly-losing positions barely distinguished "lost" (-2) from "about
+to be mated" (-3) -- no meaningful magnitude gradation at the losing
+extreme, despite real Stockfish scores in the same game ranging to
+-13 or worse just before the actual mate.
+
+**Root cause, confirmed analytically and empirically**: `raw` (the
+net's pre-sigmoid scalar) is only ever supervised through
+`sigmoid(raw/K)` against a WDL-probability target -- never regressed
+against a centipawn/pawn target directly. That sigmoid saturates hard
+once a position is clearly decided; with this corpus's originally
+fitted `K=2.0659`, `sigmoid(8/K)=0.9796` vs `sigmoid(13/K)=0.9982` --
+a target spread of under 0.02 for a 5-pawn difference in the real
+score, leaving almost no training gradient to differentiate them. No
+value of `lambda` (the `p_search`/`r_outcome` blend) fixes this, since
+both blend candidates are already post-sigmoid probabilities.
+
+**What did NOT fix it, tried in order, each measured at a medium
+scale (500k positions) before committing further**:
+1. An auxiliary `score_probe` head (nnue_model.py, same mechanism as
+ `material_probe` -- a separate `Linear` on the shared accumulator,
+ trained with its own MSE loss against the real Stockfish score).
+ Modest, inconsistent improvement -- its gradient only ever reaches
+ the *shared embedding tables*, never `fc1`/`fc2`/`fc3`, so nothing
+ forces the actually-exported path to widen its range.
+2. A direct `MSE(raw, clipped_score)` term added to the main loss
+ (replacing `score_probe` -- puts gradient straight on `fc1-3`).
+ Also modest/inconsistent across weights (0.001, 0.005) and loss
+ shapes (plain MSE vs. Huber) -- sometimes better on the winning
+ side, not reliably better on the losing side that actually
+ prompted this investigation.
+3. **The actual fix: widen K itself.** Measured `raw` vs. real score
+ at two K values (2.07, 5.0) on the same held-out sample; the ratio
+ `slope/K` came out nearly identical at both points (0.3577 vs.
+ 0.3574) -- strong confirmation that `raw` scales linearly with K
+ as theory predicts, letting the right K be solved for directly:
+ `K* = 1/0.3575 ≈ 2.8`. Confirmed by actually training at K=2.8:
+ aggregate slope landed at 1.0005 (vs. 0.74 at the original fit),
+ with per-bucket magnitude tracking the real score far more closely
+ across the whole severity range, no overshoot the way K=5.0
+ produced. Cross-checked against Stockfish's own documented NNUE
+ training write-up (`scaling_factor≈410cp=4.1 pawns`) -- our
+ corpus's own re-fit (2.8) came in lower, a reminder that this
+ project's corpus-specific fit isn't automatically wrong just for
+ disagreeing with a different engine's number, but also isn't
+ assumed right just for being "ours."
+4. Also adopted: `--loss-exponent 2.5` (Stockfish's own documented
+ practice, already wired as `powered_loss()`'s `--loss-exponent`,
+ just not previously used) -- `|error|^2.5` instead of plain MSE in
+ WDL-probability space, which shrinks every error (since errors are
+ <1) but shrinks small ones proportionally more, concentrating
+ gradient on the worst-wrong predictions. Small, not dramatic,
+ improvement on top of the K fix; adopted since it's free.
+
+**Real residual tradeoff, not a flawless fix**: after K=2.8/exponent
+2.5, aggregate calibration is essentially exact (slope ~1.0), but
+there's genuine non-linearity left (overshoot at the extremes,
+undershoot mid-range) and per-position noise went up slightly (mean
+abs error 1.78 vs. 1.57 pawns on the full-scale retrain's held-out
+sample) -- widening K to fix systematic compression also amplifies
+whatever per-position imprecision the net already had, since the same
+logit-space noise now maps to a bigger pawn-space error. Not treated
+as disqualifying, but recorded so a future reader doesn't expect this
+to be perfect.
+
+**Keeping this stable across future retrains -- the real answer isn't
+"just reuse the same K"**: tested directly, holding K fixed within one
+run still left measurable residual non-linearity/noise -- a fixed
+*recipe* (K/lambda/loss-exponent) is not the same as a fixed *output
+scale*, since training variance (data, convergence) shifts the
+empirical raw-to-score mapping even with an identical recipe. Built
+instead:
+- `export_nnue_weights.py`'s new `CalibrateScoreMapping`: fits a fresh
+ affine correction `(a, b)` via OLS, `real_score ≈ a + b*raw`,
+ against the HOLDOUT tensor dir (not train -- an honest fit needs
+ positions the checkpoint didn't see), every single export. Baked
+ into a new file format version, `NNU4` (magic bump from `NNU3`):
+ two extra float32 fields, `CALIB_A`/`CALIB_B`, read by
+ `nnueeval.c`'s `NnueLoadWeights` and applied in both `NnueForward`
+ (`(CALIB_A + CALIB_B*fOut) * 100.0f`, replacing the old bare
+ `fOut * 100.0f`) and `NnueProbeScore` (same correction, since that
+ function's own doc comment promises it stays directly comparable to
+ `NnueForward`). This decouples engine search-tuning from needing the
+ raw net's internal scale to stay perfectly constant forever -- the
+ centipawn numbers search actually consumes stay calibrated by
+ construction, regenerated fresh every generation, regardless of how
+ much the underlying raw net drifts.
+- Target for this fit is deliberately Stockfish's own score (the same
+ ground truth used throughout this project), not game outcome and
+ not HCE's scale -- outcome is noisier per-position by design
+ (lambda's whole point), and matching HCE's scale would just
+ reintroduce a version of the original compression problem in a
+ different flavor.
+- **Real, previously-undiscovered bug found and fixed in the same
+ pass**: `run_nnue_pipeline.py` never threaded `--k` through to
+ `nnue_train.py` at all -- every prior pipeline-driven production run
+ (`sf_20261019`, `sf_20261020`, `sf98k`/the then-current production
+ net) silently trained its loss against `nnue_train.py`'s own
+ `DEFAULT_K=1.3123` (a Phase-A placeholder), completely independent
+ of whatever K `fit_k_and_label.py` actually fit for that corpus's
+ own `p_search` labels. `--k` is now a required argument to
+ `run_nnue_pipeline.py` (no silent default) specifically so this
+ can't happen again unnoticed.
+
+**Lazy-eval probe now refit automatically too**: new
+`annotate_pgns/fit_lazy_probe.py` (ridge regression on the trained
+net's own accumulator, targeting that same checkpoint's raw
+`forward()` output pre-calibration, matching `NnueProbeScore`'s own
+convention) wired into `run_nnue_pipeline.py` as stage 8, run
+unconditionally after export -- every retrain now produces a fresh,
+matching probe file alongside the `.nnue` weights, written to
+`<outfile stem>_probe.bin`. The real search-node-traffic margin-safety
+calibration (next item) is deliberately **not** automated the same
+way -- too expensive to justify every generation, and not yet
+established that it needs to be redone every generation either (see
+below).
+
+**Full retrain done with the fixed recipe**: pooled all three
+Stockfish self-play PGN files (`stockfish_self_play_training_data{,.0,
+.1}.pgn`, ~120k games total), `lambda=0.99`, `K=2.8`,
+`loss-exponent=2.5`, `score-loss-weight=0` (the score_probe/direct-term
+experiments above were abandoned in favor of the K fix). 10,484,835
+training positions, 1,163,529 held out. Early-stopped at epoch 20 (5
+epochs past the best, epoch 15, per `--patience 5`), `holdout_mse=
+0.00457`. Output: `annotate_pgns/runs/sf_20261020_recal/
+nnue_weights.nnue` (+ `nnue_weights_probe.bin`).
+
+**Margin-safety calibration re-run for real** (not the offline-sample
+proxy that was all that existed before): 16 parallel shards of 12,500
+positions each (200,000 total) from `tests/cross_sectional_200k.ep_`
+at `sd 6` -- same protocol as the original 2026-10-15 calibration,
+re-run because that one was fit against an earlier net generation,
+before this recalibration widened the net's raw output scale; the old
+margins (`{180, 200, 190, 180, 170, 160, 150, 150}`) would have been
+unsafe (too tight) against the new, wider-scale net's real swings.
+New `iNnueLazyMarginByMatBucket` (p98 per material bucket, ~100-435M
+real search-node swing samples per bucket): `{300, 320, 410, 640, 770,
+770, 540, 230}`, now live in `eval.c`. New tooling:
+`annotate_pgns/aggregate_marginsafety.py` sums the raw per-shard
+histograms (`eval.c`'s `PersistMarginSafetyHistogramsToFile` format)
+bucket-and-bin-wise and computes exact combined percentiles -- shard
+percentiles don't combine validly on their own, only the raw counts
+do.
+- **Side finding**: NNUE super-lazy (material-only, no net at all)
+ measured much more competitive than the earlier "too wild to use"
+ verdict suggested -- p98 `{420, 450, 520, 730, 860, 850, 640, 290}`,
+ consistently ~30-40% looser than the real probe but the same order
+ of magnitude, not wildly worse. Likely explanation: the earlier
+ verdict was measured against the *old, uncalibrated* `NnueForward`
+ output, so comparing a stable-scale quantity (material) against a
+ compressed/wrong-scale one (the old net) spuriously inflated the
+ apparent disagreement. Not re-enabled by default -- `g_fUseNnueLazyProbe`
+ and `g_fUseNnueSuperLazy` both remain opt-in flags; flipping either
+ on by default is a strength/speed tradeoff decision that needs its
+ own match-based validation, not something to bundle into a
+ calibration-table update.
+- **Open question, deliberately not resolved**: whether this expensive
+ live-search-traffic pass needs re-running every future generation,
+ or whether a cheap offline-residual proxy (same mechanism as
+ `CalibrateScoreMapping`, just also recording percentiles) tracks it
+ closely enough to trust between occasional live re-checks. Not yet
+ tested head-to-head; treat the live pass as the trusted source until
+ that comparison is actually done.
+
+**Validation: real, independent movement on the LOP benchmark**
+(`tests/lop/` -- eval/move-agreement against a real Stockfish reference
+on a fixed 10,000-position set, much cheaper than a match -- see that
+section's own history below). Prior checkpoints (pre-recalibration):
+87.089%/87.076% eval correlation, 46.800% move correlation --
+statistically indistinguishable from classic HCE's own 87.064%/46.830%
+on the same benchmark. This recalibrated net: **90.319% eval
+correlation, 48.550% move correlation** -- the first real movement on
+this benchmark all session, and the basis on which this net was
+promoted to production (see below).
+
+**Status correction on match results -- the "has not won a single
+head-to-head match" note elsewhere in this document is now stale**:
+against the *pre-recalibration* net, a real `sd 10`-matched self-play
+run (`--sd 10`, isolating "is the net smarter per node" from the nps
+handicap) produced LLR=+2.28 against an H1 bound of +2.94, score 0.635
+over 360 games (`-72 =119 +169`), climbing every update, halted early
+as conclusive. NNUE now soundly beats classic HCE at matched search
+depth -- this was not true when the adoption-criterion section below
+was written. At matched real time (`--st 4`), the same
+pre-recalibration net was close to even (score 0.447, LLR -0.37,
+inconclusive, consistent with the known ~3.3x nps gap and
+not-yet-recalibrated search margins costing it ~3 plies of depth) --
+**the recalibrated net in this status update has not been st4-tested
+yet**; that is the planned next step, separate from this work.
+
+**Promotion policy changed, 2026-10-20**: `head_reference/nnue/
+promote.sh` now accepts LOP evidence (`--lop-baseline-log`/
+`--lop-candidate-log`, parsing `compare_lop`'s own stdout) as an
+alternative to a match log, gated on the candidate's eval correlation
+strictly exceeding the baseline's (no `--allow-negative` equivalent --
+if LOP doesn't even show improvement, it isn't evidence of anything).
+Decision, not just convenience: requiring a full self-play match for
+every incremental retrain (more data, same recipe) is impractical --
+a genuine few-Elo improvement is often not reliably resolvable in a
+match without a very large game count, and this project's own history
+(ECM solve-rate improving while early NNUE nets still lost every real
+match, see below) already shows a cheap static benchmark isn't
+*proof* of match strength either -- match evidence has its own noise
+problems at this scale, it isn't pure ground truth being traded away
+for convenience. LOP is accepted as the *routine* gate specifically
+because requiring a match every time is the impractical alternative,
+not because LOP has been shown to track match strength reliably yet.
+An occasional real match is still recommended periodically to confirm
+LOP-based promotions haven't quietly drifted away from actual playing
+strength -- this recalibrated net was itself promoted on LOP evidence
+alone (see `head_reference/nnue/current/metadata.json`), with a real
+match against the previous baseline still outstanding.
+
+## Status (2026-10-20, continued): lazy probe measured net-negative on
+## real search traffic, super-lazy confirmed a clean win and now
+## defaults on
+
+Follow-up to the "Engine calibration checklist" section above, picking
+up item 1's lazy-eval fast path -- the goal was to actually turn the
+two gates on (both were built and opt-in per the recalibration section
+just above) and measure real before/after numbers, not just trust the
+earlier margin-safety calibration as sufficient justification on its
+own.
+
+**Methodology**: `ecm_ringers`/`ecm_confident_quick`/`ecm_hard_quick`
+(the three curated suites), each at both `sd 10` (fixed depth) and
+`sn 5000000` (fixed node budget), run as a clean baseline (gates off),
+then with only the lazy probe on, then with only super-lazy on, all
+against the same `eval.nnue` weights -- never mixed/parallel-vs-
+sequential in a way that could introduce load-noise into the
+comparison (confirmed box load ~4.3 on a lightly-loaded multi-core
+machine throughout; `sd`/`sn` hold depth/node-budget fixed by
+construction regardless of load, so nps and solve-count differences
+here are real, not noise).
+
+**Baseline** (gates off): 108/191 solved across all three suites at
+`sd 10` (7/11, 73/90, 27/90), 117/191 at `sn 5000000` (8/11, 78/90,
+31/90); nps 479k-539k depending on suite/mode.
+
+**Lazy probe alone, at the real p98 margin table (`{300, 320, 410,
+640, 770, 770, 540, 230}` by material bucket, from the real
+`cross_sectional_200k.ep_`/16-shard run)**: a clear net LOSS --
+**nps down 3-7% in every configuration**, and fewer solves in 4 of 6
+configs (ringers 6/11 and 7/11 vs baseline's 7/11 and 8/11;
+confident_quick 70/90 flat/down; hard_quick 24/90 and 28/90 vs 27/90
+and 31/90). Not explained by the probe being expensive -- see the
+latency measurement below.
+
+**Direct per-call latency measurement** (new instrumentation, piled
+into the existing `EVAL_TIME` build profile rather than a new flag --
+`TIMED_EVAL_CALL` wraps the `NnueProbeScore()`/`NnueForward()` call
+sites in `eval.c`'s production NNUE branch, accumulating into two new
+always-present-but-only-`EVAL_TIME`-populated `COUNTERS.tree` fields,
+`u64CyclesNnueProbeCall`/`u64CyclesNnueForwardCall`, plus a new
+`u64NnueProbeAttempts` counter -- unlike `u64LazyEvals`, which only
+counts *hits*, this counts every attempt, hit or miss, since the gate
+pays its cost on every attempt and only recoups it on a hit).
+Reported in `root.c` right after the (NNUE-irrelevant, classic-HCE-
+only) `LAZY_EVAL` exit-breakdown block, with the measurement's own
+~38-cycle rdtsc-pair overhead printed alongside rather than silently
+subtracted, so the raw numbers can be sanity-checked against it:
+
+```
+NnueProbeScore: ~272 cyc/call (net ~235 after overhead)
+NnueForward: ~4100 cyc/call (net ~4060 after overhead)
+```
+
+**A clean 15:1 ratio, extremely consistent across all 11 `ecm_ringers`
+positions (272±3 cyc every time)** -- the probe itself is cheap and
+behaving exactly as designed. Implied breakeven hit rate: `272/4100 ≈
+6.6%`. The docs' earlier 2026-10-16 measurement (before the K/lambda
+rescale) claimed a 14-23% real hit rate, comfortably above breakeven.
+
+**But the real hit rate on this suite, at the current (post-rescale)
+p98 table, is ~1.0% overall** (71,022 hits / 7,100,023 attempts on
+`ecm_ringers`, per-position rates ranging 0.03%-6.98%) -- an order of
+magnitude under breakeven. Expected per-call cost with the gate on:
+`272 + 0.99*4100 ≈ 4331` cyc vs. `4100` cyc with it off -- a ~5.6%
+per-call slowdown, matching the measured nps regression almost
+exactly. **Root cause, not just a symptom**: the p98 margin table is
+calibrated purely for *safety* (bounding `|NnueForward() -
+NnueProbeScore()|` error to the 98th percentile), a completely
+different statistic from *how close typical nodes sit to their own
+alpha/beta window*, which is what actually determines hit rate. A
+margin can be a perfectly correct p98 safety bound and still almost
+never let the gate fire if real windows at that material bucket are
+typically narrower than the error the probe needs room for -- which
+is apparently true for most of this table's buckets (only bucket 7's
+230 happens to land in useful territory; the large buckets, 640-770,
+are far out past where any real window would resolve a cutoff).
+
+**Empirical margin-vs-hit-rate sweep** (flat override across all
+buckets via `nnue lazymargin <cp>`, `ecm_ringers`/`sd 10` only -- fast
+enough to run live rather than needing a real calibration pass):
+
+| flat margin | hit rate | solved/11 | nps |
+|---|---|---|---|
+| 770 | 0.00% | 6 | 509k |
+| 400 | 1.80% | 6 | 510k |
+| 230 | 16.0% | 6 | 538k |
+| 150 | 34.9% | 5 | 582k |
+| 100 | 45.0% | 4 | 632k |
+| 70 | 61.6% | 4 | 696k |
+| 50 | 67.9% | 3 | 775k |
+| 30 | 78.9% | 2 | 896k |
+| 15 | 87.9% | 2 | 1024k |
+
+Smooth tradeoff curve, no cliff -- the knee sits around margin≈200-230,
+where nps is clearly improving (538k, a real win over baseline) with
+no solve cost yet; solves start falling steadily from 150 down. This
+is genuine evidence a per-bucket table *could* be built to win cleanly
+-- it just isn't the p98 table, which was never tuned for this
+objective at all.
+
+**p95 recalibration attempt -- inconclusive, not a real retest**:
+tried to avoid a full hour+ `cross_sectional_200k.ep_` re-collection by
+recomputing percentiles from already-persisted histogram data. Found
+`eval.c`'s `PersistMarginSafetyHistogramsToFile` writes to
+`/var/tmp/typhoon/marginsafety_raw_histograms.txt` by default, and
+`annotate_pgns/aggregate_marginsafety.py` keeps the full binned
+histogram (not just the percentiles it prints), so an arbitrary
+percentile is cheap to recompute *if the real per-shard files still
+exist*. **They don't** -- only one small leftover file was found
+(~1.9M total `nnue_lazy` samples across buckets vs. the real run's
+~100-435M/bucket; almost certainly one shard or an early smoke test,
+not the aggregated dataset), and a project-wide `find -name "*hist*"`
+turned up nothing else NNUE-related (one old, unrelated classic-HCE
+residual-calibration file from a `guess2_30k` run). Computed p95 from
+this small/wrong-provenance file anyway as a fast, low-confidence
+stand-in (`{180, 240, 520, 530, 670, 720, 480, 310}` by bucket) and
+re-ran all three suites, both depths, with the lazy probe on this
+table. **Result was noisier than the p98 run, not clearly better**:
+some solve counts recovered (ringers back to 8/11 at `sn5M`,
+hard_quick up to 26/90 and 33/90), but nps actually *dropped* in
+several configs relative to p98 (`sn5M` confident_quick 504k vs p98's
+537k; hard_quick 483k vs 512k) -- backwards from what a looser margin
+should do, strongly suggesting the small sample's per-bucket numbers
+don't reliably reflect the real distribution. **Not trusted as a real
+retune** -- a legitimate hit-rate-targeted per-bucket calibration
+still needs the full `cross_sectional_200k.ep_` collection (or
+equivalent), this time explicitly measuring node-level alpha/beta
+proximity (what determines hit rate), not just probe-vs-forward error
+(what the existing harness measures, which only bounds safety).
+
+**Super-lazy alone, at the real p98 margin table (newly built this
+session -- previously a single flat runtime override with no
+per-bucket fit at all, `g_iNnueSuperLazyMargin`)**: measured a clean
+win across the board. `sd 10`: ringers 5/11 (down from 7, but ringers
+is only 11 positions -- noisy), confident_quick 70/90 (down from 73,
+flat vs lazy's own 70), hard_quick 26/90 (close to baseline's 27);
+nps 478k-504k, roughly matching or beating baseline (479k-502k).
+`sn 5000000`: ringers 6/11 (down from 8), confident_quick **79/90**
+(up from 78), hard_quick **33/90** (up from 31); nps **540k-601k**,
+a genuine 1-12% win over baseline (514k-539k) in every suite. Net
+read: real nps win, solve-count effect mixed but mostly neutral-to-
+positive outside the small `ecm_ringers` suite (where single-position
+flips swing the ratio a lot) -- per the user's own judgment, ECM
+solve-count noise at this suite size isn't treated as a correctness
+signal worth chasing into `tests/ECM.md` lookups; the nps win is the
+real, load-bearing result.
+
+**Same-bucketing confirmed**: super-lazy's own margin-safety
+calibration (`RecordNnueSuperLazyMarginSafetySwing`,
+`g_uNnueSuperLazyMarginSwingMax[BASE_MARGIN_MAT_BUCKETS]` etc.) uses
+the identical `_MaterialBucket()` (8 buckets, combined-army-scaler
+width 8) as the lazy probe's own table.
+
+**No bucket excluded outright** (unlike classic HCE's own super-lazy,
+which structurally excludes material bucket 0 -- bare-king/pawn-
+dominated endgames -- per a specific *exceeded-rate* finding, 6.8%
+blow-through from pawn-structure blindness, since HCE's super-lazy
+runs before `_EvalPawns` entirely). NNUE's measured p98 swing by
+bucket, `{420, 450, 520, 730, 860, 850, 640, 290}`, has bucket 0 as
+the *second-lowest* value, not an outlier -- nothing in the data says
+to disable it. Caveat: no exceeded-rate-specific check (as opposed to
+plain p98 magnitude) has been run for NNUE's super-lazy the way it was
+for HCE's, so this is "nothing visible says disable a bucket," not
+"confirmed safe at every bucket."
+
+**Decision**: lazy probe gate code kept live (cheap to leave dormant,
+and the latency math + margin sweep both say a real win is plausible
+with the right margins -- just not demonstrated yet), but
+`g_fUseNnueLazyProbe` stays `FALSE` by default (`nnueeval.c`), margin
+table reverted to the real p98 values (the p95 guess is preserved only
+as a commented-out, documented-as-unreliable alternative). Super-lazy
+promoted to default-on (`g_fUseNnueSuperLazy = TRUE`, `nnueeval.c`),
+using its own new per-bucket `iNnueSuperLazyMarginByMatBucket` table
+(`eval.c`) rather than the old flat-only override, which remains
+available via `g_iNnueSuperLazyMargin >= 0` for experimentation. Both
+gates only take effect when `g_fUseNnueEval` is also on.
+
+**Follow-up, not yet done**: a real hit-rate-targeted per-bucket lazy
+probe retune, against ordinary (non-adversarial-tactical) search
+traffic such as `twic_sample.ep_` rather than ECM -- the current
+margin-vs-hit-rate sweep and the docs' original 14-23%-hit-rate/
+10.7%-nps-win measurement were on different corpora, and ECM's
+narrower forcing-line alpha/beta windows may themselves suppress hit
+rate in a way that doesn't generalize to normal play. Also worth
+measuring: an exceeded-rate check for NNUE super-lazy's bucket 0,
+matching the specific diagnostic that led HCE to exclude it, rather
+than relying on p98 magnitude alone.
diff --git a/src/eval.c b/src/eval.c
index de59dd3..0f2a556 100644
--- a/src/eval.c
+++ b/src/eval.c
@@ -7239,23 +7239,86 @@ Return value:
if ((TRUE == g_fUseNnueEval) && (TRUE == g_fNnueWeightsLoaded))
{
// p98 of |NnueForward() - NnueProbeScore()| per material
- // bucket. Superseded 2026-10-15: originally derived from a
- // 91M-sample population over three curated tactical suites
- // (sd 10) -- replaced same day by a far larger and more
- // representative 3.09-BILLION-sample population, 16 parallel
- // shards of 12,500 positions each (200,000 total) from
- // tests/cross_sectional_200k.ep_ at sd 6. Per-suite tactical
- // positions are a narrow, biased slice (selected for sharp
- // tactics); the cross-sectional pool's much broader position
- // variety surfaced real outlier swings the narrower sample
- // never saw (max up to 534cp here vs ~472cp before) while
- // landing close to the same per-bucket numbers overall (mostly
- // 0-10cp different) -- see docs/NNUE.md's "Lazy-evaluation
- // tiering plan". Same bucketing as _MaterialBucket (combined
- // army scaler, width BASE_MARGIN_MAT_WIDTH).
+ // bucket. Superseded 2026-10-20: the previous table (180, 200,
+ // 190, 180, 170, 160, 150, 150) was fit against an earlier net
+ // generation, before that day's K/lambda/loss-exponent
+ // recalibration (see docs/NNUE.md's "keeping this stable
+ // across retrains" conversation) widened the net's raw output
+ // scale to actually track Stockfish's real score magnitude --
+ // the old, narrower-scale margins would have been unsafe (too
+ // tight) against the new, wider-scale net. Re-measured the
+ // same way (16 parallel shards of 12,500 positions each,
+ // 200,000 total, from tests/cross_sectional_200k.ep_ at sd 6,
+ // ~100-435M swing samples per bucket) against the recalibrated
+ // net (annotate_pgns/runs/sf_20261020_recal/nnue_weights.nnue)
+ // and its freshly-refit probe -- see
+ // annotate_pgns/fit_lazy_probe.py and aggregate_marginsafety.py.
+ // Must be re-measured again for any future net whose raw
+ // output scale changes meaningfully (a new K, a materially
+ // different training recipe) -- not assumed to stay valid
+ // forever just because the recipe nominally stayed the same
+ // (see that same conversation for why). Same bucketing as
+ // _MaterialBucket (combined army scaler, width
+ // BASE_MARGIN_MAT_WIDTH).
+ // 2026-10-20 status: lazy probe gate measured net NEGATIVE on
+ // real ECM search traffic at this margin table (p98, from the
+ // real cross_sectional_200k.ep_/16-shard run) -- hit rate ~1%
+ // overall, far under the ~6.6% breakeven implied by
+ // NnueProbeScore's own ~272cyc vs NnueForward's ~4100cyc
+ // latency (measured directly, see root.c's EVAL_TIME-gated
+ // "NNUE eval primitive latency" report). A p95 guess (NOT from
+ // this real run -- its per-shard raw histograms no longer
+ // exist on disk; computed instead from a much smaller
+ // leftover file of unknown/mismatched provenance, see git
+ // history) didn't clearly fix it either, and came back noisier
+ // than p98 on nps in several configs. g_fUseNnueLazyProbe
+ // stays FALSE by default (nnueeval.c) pending a real
+ // hit-rate-targeted per-bucket retune (not yet done -- see
+ // docs/NNUE.md) measured against ordinary search traffic
+ // (twic_sample.ep_-like), not just ECM's narrower tactical
+ // windows. Code kept live (not deleted) since the mechanism is
+ // cheap to leave dormant and the latency math says a real win
+ // is plausible with the right margins -- just not demonstrated
+ // yet. This table is the one real, validated data point (p98,
+ // safety-only objective) and is what a future retune should
+ // start from and compare against.
static const SCORE iNnueLazyMarginByMatBucket[BASE_MARGIN_MAT_BUCKETS] =
{
- 180, 200, 190, 180, 170, 160, 150, 150,
+ 300, 320, 410, 640, 770, 770, 540, 230,
+ };
+ // p95 experiment (2026-10-20) -- see the status comment above
+ // for why this isn't trusted as a real calibration. Kept here
+ // for reference/reproducibility, not as a ready-to-flip
+ // alternative.
+ // static const SCORE iNnueLazyMarginByMatBucketP95Guess[BASE_MARGIN_MAT_BUCKETS] =
+ // {
+ // 180, 240, 520, 530, 670, 720, 480, 310,
+ // };
+
+ // 2026-10-20: NNUE super-lazy's own per-bucket table, same
+ // calibration run (cross_sectional_200k.ep_, sd 6, same
+ // _MaterialBucket) as iNnueLazyMarginByMatBucket above --
+ // p98 of |NnueForward() - pos->iMaterialBalance[uToMove]|.
+ // Previously a single flat runtime override
+ // (g_iNnueSuperLazyMargin) with no per-bucket fit at all; per
+ // docs/NNUE.md's "Side finding" (2026-10-20), NNUE super-lazy's
+ // real measured swing is only ~30-40% looser than the real
+ // probe's, same order of magnitude, not "too wild to use" as
+ // an earlier (differently-scaled-net) measurement had
+ // suggested. No bucket excluded outright the way classic
+ // HCE's super-lazy excludes material bucket 0 (eval.c's
+ // fBareKingEitherSide/uSuperLazyMatBucket==0 guard) --
+ // that exclusion was driven by a specific *exceeded-rate*
+ // finding (6.8% blow-through from pawn-structure blindness,
+ // since HCE's super-lazy runs before _EvalPawns), not by p98
+ // magnitude alone, and NNUE's bucket 0 p98 here (420) is in
+ // fact the second-lowest of the 8, not an outlier. No
+ // equivalent exceeded-rate check has been run for NNUE's
+ // super-lazy yet, though -- this is "nothing visible says
+ // disable a bucket," not "confirmed safe at every bucket."
+ static const SCORE iNnueSuperLazyMarginByMatBucket[BASE_MARGIN_MAT_BUCKETS] =
+ {
+ 420, 450, 520, 730, 860, 850, 640, 290,
};
#ifdef CALIBRATE_MARGIN_SAFETY
@@ -7307,20 +7370,27 @@ Return value:
// entirely -- this is the actual speed win. Otherwise (gate
// disabled, probe not loaded, or inconclusive), fall through
// to the full pass exactly as before this feature existed.
- // Super-lazy pre-filter, experimental -- 2026-10-16: cheapest
- // possible check, pure material balance, no net evaluation at
- // all. Previously measurement-only (CALIBRATE_MARGIN_SAFETY
- // above records the real swing but always used an infinite
- // sentinel margin, i.e. never actually gated). Deliberately
- // loose/uncalibrated here per the "safety doesn't win games"
- // experiment -- g_iNnueSuperLazyMargin is a flat runtime
- // override (see nnueeval.c), not a per-bucket fit.
+ // Super-lazy pre-filter -- cheapest possible check, pure
+ // material balance, no net evaluation at all. Previously
+ // measurement-only (CALIBRATE_MARGIN_SAFETY above records the
+ // real swing but always used an infinite sentinel margin, i.e.
+ // never actually gated), then briefly a single uncalibrated
+ // flat runtime override. 2026-10-20: now a real per-material-
+ // bucket table (iNnueSuperLazyMarginByMatBucket, above),
+ // calibrated the same way/same run as the lazy probe's own
+ // table -- g_iNnueSuperLazyMargin remains available as an
+ // explicit flat override (>= 0) for experimentation, matching
+ // g_iNnueLazyMarginOverride's convention; < 0 (the default)
+ // uses the calibrated table.
if (TRUE == g_fUseNnueSuperLazy)
{
SCORE iSuperLazyScore = pos->iMaterialBalance[pos->uToMove];
+ SCORE iSuperLazyMargin = (g_iNnueSuperLazyMargin >= 0) ?
+ g_iNnueSuperLazyMargin :
+ iNnueSuperLazyMarginByMatBucket[_MaterialBucket(pos)];
- if (((iSuperLazyScore - g_iNnueSuperLazyMargin) > iBeta) ||
- ((iSuperLazyScore + g_iNnueSuperLazyMargin) < iAlpha))
+ if (((iSuperLazyScore - iSuperLazyMargin) > iBeta) ||
+ ((iSuperLazyScore + iSuperLazyMargin) < iAlpha))
{
// Feed the same aggregate counters root.c's "Eval exit
// breakdown" line reads -- otherwise that line stays
@@ -7339,7 +7409,10 @@ Return value:
}
if ((TRUE == g_fUseNnueLazyProbe) && (TRUE == g_fNnueProbeLoaded))
{
- SCORE iProbeScore = NnueProbeScore(ctx);
+ SCORE iProbeScore;
+ INC(ctx->sCounters.tree.u64NnueProbeAttempts);
+ TIMED_EVAL_CALL(ctx, u64CyclesNnueProbeCall,
+ iProbeScore = NnueProbeScore(ctx));
SCORE iMargin = (g_iNnueLazyMarginOverride >= 0) ?
(SCORE)g_iNnueLazyMarginOverride :
iNnueLazyMarginByMatBucket[_MaterialBucket(pos)];
@@ -7359,7 +7432,9 @@ Return value:
}
}
{
- SCORE iNnueScore = NnueForward(ctx);
+ SCORE iNnueScore;
+ TIMED_EVAL_CALL(ctx, u64CyclesNnueForwardCall,
+ iNnueScore = NnueForward(ctx));
INC(ctx->sCounters.tree.u64FullEvals);
ctx->sPlyInfo[ctx->uPly].uLastPositionalSource = EVAL_POSITIONAL_SOURCE_NNUE_FULL;
g_u64NnueFullEvalsByBucket[_MaterialBucket(pos)]++;
diff --git a/src/nnueeval.c b/src/nnueeval.c
index 7caadef..583c2db 100644
--- a/src/nnueeval.c
+++ b/src/nnueeval.c
@@ -45,6 +45,24 @@ CHAR g_rgfNnueFc3Weight[NNUE_L3];
float g_rgfNnueFc3Bias;
float g_rgfNnueActScale[3] = { 1.0f, 1.0f, 1.0f };
float g_rgfNnueWeightScale[3] = { 1.0f, 1.0f, 1.0f };
+// Affine correction so the engine's consumed centipawn output tracks
+// Stockfish's own real score, not the net's nominal raw WDL-sigmoid
+// scale -- see export_nnue_weights.py's CalibrateScoreMapping and the
+// 2026-10-20 "keeping this stable across retrains" conversation. A
+// fixed K/lambda/loss-exponent training recipe is NOT enough on its
+// own to guarantee the search's consumed cp scale stays constant
+// across future retrains -- training variance (data, convergence)
+// still shifts the empirical raw-to-score mapping even with an
+// identical recipe. Re-fitting and re-baking this correction into
+// every export, rather than trusting a fixed multiplier, is what
+// actually decouples engine-side search tuning from needing the raw
+// net's internal scale to stay perfectly constant forever. Defaults
+// (0,1) are the identity transform -- the pre-2026-10-20 behavior --
+// so an old build loading a newer file (or vice versa, before the
+// dimension check below even runs) degrades to the old bare-*100
+// convention rather than silently miscalibrating.
+float g_fNnueCalibA = 0.0f;
+float g_fNnueCalibB = 1.0f;
FLAG g_fNnueWeightsLoaded = FALSE;
FLAG g_fUseNnueEval = FALSE;
@@ -86,12 +104,13 @@ Return value:
}
if ((1 != fread(szMagic, 4, 1, fp)) ||
- (0 != memcmp(szMagic, "NNU3", 4)))
+ (0 != memcmp(szMagic, "NNU4", 4)))
{
Trace("NnueLoadWeights: %s is not a valid NNUE weights file "
- "(bad magic -- expected \"NNU3\"; an older \"NNU1\"/"
- "\"NNU2\" file predates the FC head's int8 quantization "
- "and is no longer readable)\n", szFilename);
+ "(bad magic -- expected \"NNU4\"; an older \"NNU1\"/"
+ "\"NNU2\"/\"NNU3\" file predates the raw-to-score "
+ "calibration fields and is no longer readable)\n",
+ szFilename);
fclose(fp);
return FALSE;
}
@@ -130,6 +149,14 @@ Return value:
fclose(fp);
return FALSE;
}
+ if ((1 != fread(&g_fNnueCalibA, sizeof(g_fNnueCalibA), 1, fp)) ||
+ (1 != fread(&g_fNnueCalibB, sizeof(g_fNnueCalibB), 1, fp)))
+ {
+ Trace("NnueLoadWeights: %s truncated (score calibration)\n",
+ szFilename);
+ fclose(fp);
+ return FALSE;
+ }
#define READ_OR_FAIL(ptr, elsize, count) \
do { \
@@ -265,7 +292,7 @@ Return value:
g_rgfNnueFc3Bias;
}
- return (SCORE)(fOut * 100.0f);
+ return (SCORE)((g_fNnueCalibA + g_fNnueCalibB * fOut) * 100.0f);
}
@@ -420,14 +447,23 @@ FLAG g_fUseNnueLazyProbe = FALSE;
// docs/NNUE.md "paradox: safety doesn't win games" discussion.
// g_iNnueLazyMarginOverride < 0 means "use eval.c's calibrated
// per-material-bucket table as before"; >= 0 means "use this flat cp
-// value for every bucket instead". g_fUseNnueSuperLazy/
-// g_iNnueSuperLazyMargin gate a brand new, even cheaper pre-filter
-// (pure material balance, no net at all) ahead of the probe -- this
-// was previously measurement-only (see eval.c's CALIBRATE_MARGIN_SAFETY
-// block) and is now wired into the real Eval() path for the first time.
+// value for every bucket instead". g_fUseNnueSuperLazy gates an even
+// cheaper pre-filter (pure material balance, no net at all) ahead of
+// the probe. g_iNnueSuperLazyMargin follows the identical convention
+// as of 2026-10-20 (was a flat-only override before that date, with no
+// per-bucket fit at all): < 0 means "use eval.c's calibrated
+// iNnueSuperLazyMarginByMatBucket table", >= 0 means "use this flat cp
+// value for every bucket instead".
INT g_iNnueLazyMarginOverride = -1;
-FLAG g_fUseNnueSuperLazy = FALSE;
-SCORE g_iNnueSuperLazyMargin = 0;
+// 2026-10-20: super-lazy defaults ON -- measured a clean win (nps up,
+// no consistent solve-rate cost) across all three curated ECM suites
+// at both sd 10 and sn 5000000, using the calibrated per-bucket
+// iNnueSuperLazyMarginByMatBucket table (eval.c), unlike the lazy
+// probe gate just above, which stays off by default (measured net
+// negative -- see its own table's comment in eval.c for the full
+// story). Still only takes effect when g_fUseNnueEval is also on.
+FLAG g_fUseNnueSuperLazy = TRUE;
+SCORE g_iNnueSuperLazyMargin = -1;
FLAG
@@ -554,5 +590,11 @@ Return value:
fSum += v * g_rgfNnueProbeWeight[NNUE_ACCUMULATOR_SIZE + i];
}
- return (SCORE)(fSum * 100.0f);
+ // fSum approximates NnueForward's pre-calibration raw fOut (it was
+ // fit by ridge regression against exactly that target -- see
+ // docs/NNUE.md's "Lazy-evaluation tiering plan") -- apply the same
+ // (g_fNnueCalibA, g_fNnueCalibB) correction so this stays directly
+ // comparable to NnueForward's actual output, per this function's
+ // own doc comment above.
+ return (SCORE)((g_fNnueCalibA + g_fNnueCalibB * fSum) * 100.0f);
}
diff --git a/src/root.c b/src/root.c
index 47fd103..f686efa 100644
--- a/src/root.c
+++ b/src/root.c
@@ -628,6 +628,59 @@ Return value:
#endif
}
#endif
+#ifdef EVAL_TIME
+ // 2026-10-20: NNUE's own per-call latency, independent of the
+ // LAZY_EVAL block above (NNUE is a runtime g_fUseNnueEval switch,
+ // not a #ifdef LAZY_EVAL path, so it never fed that block's
+ // counters). Direct answer to "is the lazy-probe gate paying for
+ // itself": the gate costs u64CyclesNnueProbeCall on EVERY attempt
+ // (hit or miss -- u64NnueProbeAttempts, not just u64LazyEvals'
+ // hit count) and only SAVES a full-forward call on a hit, so the
+ // gate is worth it only if (hit rate) * (forward cost - probe
+ // cost) exceeds (miss rate) * (probe cost).
+ //
+ // Caveat printed alongside rather than silently baked in: each
+ // TIMED_EVAL_CALL is a pair of back-to-back SystemReadTimeStamp-
+ // Counter() (rdtsc) reads, and rdtsc itself isn't free (commonly
+ // cited around ~80 cycles, more with the serializing behavior
+ // some cores give it) -- for NnueForward's thousands of cycles
+ // that overhead is noise, but for NnueProbeScore's much smaller
+ // 512-wide dot product it may not be. Measure the pair's own
+ // overhead right here (an empty back-to-back read, no call in
+ // between) so the raw numbers below can be read against it rather
+ // than taken as exact.
+ {
+ UINT64 u64OverheadStart = SystemReadTimeStampCounter();
+ UINT64 u64OverheadEnd = SystemReadTimeStampCounter();
+ UINT64 u64PairOverhead = u64OverheadEnd - u64OverheadStart;
+ UINT64 u64ProbeCyc = ctx->sCounters.tree.u64CyclesNnueProbeCall;
+ UINT64 u64ProbeAttempts = ctx->sCounters.tree.u64NnueProbeAttempts;
+ UINT64 u64FwdCyc = ctx->sCounters.tree.u64CyclesNnueForwardCall;
+ UINT64 u64FwdCalls = ctx->sCounters.tree.u64FullEvals;
+ double dProbeAvg = u64ProbeAttempts ?
+ (double)u64ProbeCyc / (double)u64ProbeAttempts : 0.0;
+ double dFwdAvg = u64FwdCalls ?
+ (double)u64FwdCyc / (double)u64FwdCalls : 0.0;
+
+ if ((0 != u64ProbeAttempts) || (0 != u64FwdCalls))
+ {
+ Trace("NNUE eval primitive latency (rdtsc pair overhead ~%"
+ COMPILER_LONGLONG_UNSIGNED_FORMAT" cyc, subtract for a "
+ "rough net estimate):\n"
+ " NnueProbeScore: %10.1f cyc/call (raw, %"
+ COMPILER_LONGLONG_UNSIGNED_FORMAT" attempts, net ~%10.1f)\n"
+ " NnueForward: %10.1f cyc/call (raw, %"
+ COMPILER_LONGLONG_UNSIGNED_FORMAT" calls, net ~%10.1f)\n",
+ u64PairOverhead,
+ dProbeAvg, u64ProbeAttempts,
+ (dProbeAvg > (double)u64PairOverhead) ?
+ (dProbeAvg - (double)u64PairOverhead) : 0.0,
+ dFwdAvg, u64FwdCalls,
+ (dFwdAvg > (double)u64PairOverhead) ?
+ (dFwdAvg - (double)u64PairOverhead) : 0.0);
+ }
+ }
+#endif
Trace("Extensions: (%u +, %u q+, %u 1mv, %u !kmvs, %u mult+, %u pawn\n"
" %u threat, %u zug, %u sing, %u endg, %u bm, %u recap)\n"
" check by amount (1/4,1/2,3/4,1 ply): %u %u %u %u\n"