Wire NNUE eval into search, build the training pipeline, and add a fast

move/eval-agreement benchmark against Stockfish (tests/lop)

Implements the architecture decisions in src/docs/NNUE.md: HalfKAv2_hm
feature set, int16/int8 quantization, per-ply accumulator backup for
king moves, and a lazy-eval-ready integration into search.c/eval.c.
Adds the full annotate_pgns/ training pipeline (extraction, K-fit/
labeling, quiet filtering, tensor building, training, export) plus an
end-to-end orchestrator (run_nnue_pipeline.py) and validation tooling
(sanity_check_net.py, compare_vs_stockfish_net.py). Adds tests/lop:
a 10k-position list-of-positions benchmark comparing engine eval rank
and move choice against pre-computed Stockfish/lc0 references, much
cheaper than a full self-play match for day-to-day progress checks.

First sd10-matched self-play result against classic HCE: LLR +2.28
(H1=+2.94), -72=119+169 over 360 games, confirming the net is
"smarter per node" at matched depth. A real-time (--st 4) match is
currently weaker (candidate loses ~3 plies of depth to a ~3.3-3.7x
nps gap against an unrecalibrated search), consistent with
NNUE.md's adoption-criterion section -- pruning/reduction constants
still need retuning for the new eval's score scale before a fair
real-time comparison is possible.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

1 files changed, 240 insertions, 35 deletions

diff --git a/src/eval_tune/match_play.py b/src/eval_tune/match_play.py
index 5033cad..34a7917 100755
--- a/src/eval_tune/match_play.py
+++ b/src/eval_tune/match_play.py
@@ -96,7 +96,13 @@ _FIRST_MOVE_BETA_RE = re.compile(
 # count, (4) the rest of the line: the SAN PV text (WalkPV, util.c), or
 # "FL --" for a fail-low line (no PV -- the move that failed low isn't
 # necessarily the move eventually played) or "<SAN> ++" for a fail-high.
-_ITER_RE = re.compile(r"^\s*(\d+)[+-]?\s+(-?\d+)\s+\S+\s+(\d+)\s*(.*)$")
+# 2026-10-17: score group widened from (-?\d+) to ([+-]?\d+) -- typhoon
+# itself never prints a leading "+" on a non-negative score (bare
+# digits), but a UCI engine driven through PolyGlot's xboard adapter
+# (e.g. Stockfish, see stockfish_xboard) DOES print an explicit "+"
+# ("23 +28 198 700479 e4 c5"), which the old pattern rejected outright
+# -- confirmed via direct regex test before this fix, not assumed.
+_ITER_RE = re.compile(r"^\s*(\d+)[+-]?\s+([+-]?\d+)\s+\S+\s+(\d+)\s*(.*)$")
 # Reading PostMoveSearchReport's trailing block used to stop either at a
 # bounded line count or at gamelist.c's "_white(move N)_"/"_black(move N)_"
 # prompt -- both unreliable: the block's line count depends on whether the
@@ -307,6 +313,13 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
     white_stats = EngineStats()
     black_stats = EngineStats()
     white_last_fmb = black_last_fmb = None
+    # Tracked across the whole engine-interaction block below so the
+    # BrokenPipeError handler (see try/except wrapping it) can report
+    # which side/ply was in flight even when the pipe dies on a write
+    # that isn't inside the per-ply move loop (e.g. one of the startup
+    # sends, or a time-control send before the first ply).
+    ply_idx = -1
+    white_to_move = None
     # Default Engine() spawns with NO --hash/--cpus at all, which means
     # a tiny 65536-entry hash table (main.c's memset-zero default) --
     # far smaller than the 256m used everywhere else in this project's
@@ -342,7 +355,8 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
         # and the default behavior of a harness in active use should not
         # change silently underneath a running experiment.
         startup = ["book name /none",
-                   "set ThinkOnOpponentsTime false"]
+                   "set ThinkOnOpponentsTime false",
+                   "set PostThrottle 0.0"]
         if post:
             startup.append("post")
         for command in startup:
@@ -416,6 +430,7 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
                 tail = im.group(4).strip()
                 if (last_iter_depth is not None and
                         last_iter_nodes is not None and
+                        last_iter_nodes > 0 and
                         iter_nodes > last_iter_nodes and
                         iter_depth > last_iter_depth):
                     iter_log_ratio_sum += (math.log(iter_nodes / last_iter_nodes)
@@ -434,10 +449,12 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
                 last_eval_score = iter_score
                 last_eval_pv = tail[:-3].rstrip() if tail.endswith(" ++") else tail
 
+            abort_reason = None
             for _ in range(400):
                 line = mover.stdout.readline()
                 if not line:
                     _dbg(tag, "EOF while scanning for move line")
+                    abort_reason = "eof"
                     break
                 _dbg(tag, f"read(move-scan): {line!r}")
                 _note_iter(line)
@@ -445,15 +462,67 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
                     # Engine resigned instead of moving -- root.c prints
                     # this and returns without ever printing "move ...",
                     # so stop reading now rather than blocking forever.
+                    # This IS a legitimate decisive result (unlike the
+                    # other abort_reason cases below) -- resignation is a
+                    # real chess outcome, not a harness/engine failure.
+                    abort_reason = "resign"
                     break
                 m = _MOVE_RE.match(line)
                 if m:
-                    move_uci = m.group(1)
+                    # root.c's "move ..." line is xboard/ICS-style
+                    # (MoveToIcs, ics.c), which prints the promotion
+                    # piece UPPERCASE ("e7e8Q") -- correct for that
+                    # protocol, but chess.Move.from_uci() below enforces
+                    # strict lowercase UCI and raises ValueError on it.
+                    # Unlowered, EVERY promotion move got misclassified
+                    # as illegal and the game aborted right as the
+                    # promoting side (almost always the winning side in
+                    # a long endgame) was about to seal the win -- this
+                    # was the actual root cause of the ~47%-corrupted
+                    # self-play corpus (docs/NNUE.md, 2026-10-17), not an
+                    # engine stall/crash. Squares/files are already
+                    # lowercase, so a blanket .lower() is safe here.
+                    move_uci = m.group(1).lower()
                     break
+            else:
+                # Read 400 lines without ever seeing "move ..." (and
+                # without EOF or resignation either) -- most likely an
+                # extremely verbose iteration output (deep search in a
+                # sparse endgame can print far more than 400 lines for a
+                # single move) exceeding this loop's hard cap while the
+                # engine is still alive and about to answer.
+                abort_reason = "line-cap-exceeded"
             if move_uci is None:
-                # side to move had nothing to play -- treat as loss for mover
-                result = 0.0 if white_to_move else 1.0
-                return result, moves, move_evals, white_stats, black_stats
+                if abort_reason == "resign":
+                    # side to move resigned -- legitimate loss for mover
+                    result = 0.0 if white_to_move else 1.0
+                    return (result, moves, move_evals, white_stats,
+                            black_stats, None)
+                # 2026-10-17: a real bug was found producing exactly this
+                # path for a non-resign reason (eof / line-cap) on a
+                # position where the mover was completely winning --
+                # resignation was disabled for this corpus, so there is
+                # no legitimate explanation for move_uci staying None
+                # other than resign. The OLD code silently recorded this
+                # as an ordinary loss for the mover, which corrupted
+                # ~47% of a self-play training corpus with inverted
+                # results (see docs/NNUE.md). Never do that again: signal
+                # the caller with an aborted (result=None) game instead
+                # of guessing a winner, and capture enough context here
+                # to investigate later without needing to reproduce it
+                # live.
+                exit_code = mover.proc.poll()
+                abort_info = {
+                    "reason": abort_reason,
+                    "mover": "white" if white_to_move else "black",
+                    "ply": ply_idx,
+                    "fen_before_ply": board.fen(),
+                    "mover_exit_code": exit_code,
+                    "mover_alive": exit_code is None,
+                }
+                _dbg(tag, f"ABORT (not scored): {abort_info}")
+                return (None, moves, move_evals, white_stats, black_stats,
+                        abort_info)
 
             # PostMoveSearchReport's stats block trails the "move" line
             # (confirmed against root.c and real output), not the other
@@ -512,14 +581,32 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
                 else:
                     black_last_fmb = ply_fmb
 
+            move = None
             try:
                 move = chess.Move.from_uci(move_uci)
-                if move not in board.legal_moves:
-                    result = 0.0 if white_to_move else 1.0
-                    return result, moves, move_evals, white_stats, black_stats
+                illegal = move not in board.legal_moves
             except ValueError:
-                result = 0.0 if white_to_move else 1.0
-                return result, moves, move_evals, white_stats, black_stats
+                illegal = True
+            if illegal:
+                # The engine printed a "move ..." line, but it wasn't a
+                # legal move -- same reasoning as the move_uci-is-None
+                # case above: a real engine bug (producing garbage)
+                # deserves the same abort-not-score treatment, not a
+                # confidently-recorded loss with no diagnostic trail.
+                exit_code = mover.proc.poll()
+                abort_info = {
+                    "reason": "illegal-move",
+                    "mover": "white" if white_to_move else "black",
+                    "ply": ply_idx,
+                    "fen_before_ply": board.fen(),
+                    "illegal_move_uci": move_uci,
+                    "mover_exit_code": exit_code,
+                    "mover_alive": exit_code is None,
+                }
+                _dbg(tag, f"ABORT (not scored): {abort_info}")
+                return (None, moves, move_evals, white_stats, black_stats,
+                        abort_info)
+            assert move is not None  # illegal is False only when the try succeeded
             board.push(move)
             moves.append(move_uci)
             score_white = None
@@ -536,7 +623,7 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
             result = 0.5
         else:
             result = 1.0 if outcome.winner == chess.WHITE else 0.0
-        return result, moves, move_evals, white_stats, black_stats
+        return result, moves, move_evals, white_stats, black_stats, None
 
 
 def _format_eval_comment(score_white, pv, depth):
@@ -576,7 +663,15 @@ def build_pgn_game(start_fen, moves, result_white, white_name, black_name,
     game.headers["FEN"] = start_fen
     game.headers["Date"] = f"{date.today()}"
     game.setup(board)
-    result_str = {1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white]
+    # result_white is None for an ABORTED game (play_one_game's mover
+    # failed to produce a legal move for a non-resign reason -- see its
+    # abort_info return value) -- write a standard "unknown result" PGN
+    # tag rather than guessing a winner. fit_k_and_label.py's
+    # result_to_white_score() already treats "*" as "unknown, skip this
+    # game" for training purposes, so this alone keeps an aborted game
+    # out of any future training corpus without touching that script.
+    result_str = ({1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white]
+                  if result_white is not None else "*")
     game.headers["Result"] = result_str
     node = game
     for i, move_uci in enumerate(moves):
@@ -888,6 +983,48 @@ def main():
                           "Without this, local_ebf reads 0.000 -- the "
                           "engines never print those lines because "
                           "Engine.__init__ sends 'nopost'.")
+    ap.add_argument("--candidate-name", default="candidate",
+                     help="PGN White/Black name recorded for the "
+                          "candidate engine (default: 'candidate'). "
+                          "2026-10-16: set this to something identifying "
+                          "the eval/config actually used (e.g. "
+                          "'candidate_nnue_loose') whenever the engine "
+                          "isn't a plain HCE build -- self_play_games.pgn "
+                          "is reused as NNUE training data later, and a "
+                          "generic 'candidate' name makes it impossible "
+                          "to tell which games came from a differently-"
+                          "calibrated evaluator (whose scores shouldn't "
+                          "be trusted as training labels) without "
+                          "externally remembering run dates.")
+    ap.add_argument("--baseline-name", default="baseline",
+                     help="PGN White/Black name recorded for the "
+                          "baseline/head engine (default: 'baseline'); "
+                          "same reasoning as --candidate-name.")
+    ap.add_argument("--no-color-swap", action="store_true",
+                     help="play each sampled opening ONCE instead of "
+                          "twice with colors swapped between head/"
+                          "candidate. The swap exists to cancel out "
+                          "color-dependent FIRST-MOVE ADVANTAGE in an "
+                          "Elo-gating match; it is pure waste -- a byte-"
+                          "for-byte duplicate game, just relabeled --  "
+                          "whenever head_engine and candidate_engine are "
+                          "the SAME binary (as every self-play "
+                          "training-data run via "
+                          "hce_training_corpus.sh is), because search "
+                          "here is deterministic (--cpus 1) given a "
+                          "fixed FEN+side-to-move, so swapping which "
+                          "name is 'white'/'black' changes no move "
+                          "actually played. Confirmed empirically "
+                          "2026-10-16: consecutive game pairs in "
+                          "nnue_self_play_training_data.pgn were exact "
+                          "duplicates (identical FEN, identical move "
+                          "list). Use this flag for self-play corpus "
+                          "generation -- sample --games distinct "
+                          "openings and play each once, rather than "
+                          "--games/2 openings played twice -- and keep "
+                          "the swap (the default) for real head-vs-"
+                          "candidate Elo gating, where two different "
+                          "binaries need it to cancel color bias.")
     ap.add_argument("--max-plies", type=int, default=200)
     ap.add_argument("--scratch", default="/usr/local/tmp/typhoon_match")
     ap.add_argument("--log", default="match_history.log")
@@ -968,13 +1105,16 @@ def main():
     head_engine = str(Path(args.head_engine).resolve())
     candidate_engine = str(Path(args.candidate_engine).resolve())
 
-    n_pairs = args.games // 2
-    openings = sample_openings(args.pgn, n_pairs, seed=args.opening_seed)
-    n_games = len(openings) * 2
+    if args.no_color_swap:
+        n_openings = args.games
+    else:
+        n_openings = args.games // 2
+    openings = sample_openings(args.pgn, n_openings, seed=args.opening_seed)
 
     jobs = []
     for fen in openings:
-        for candidate_is_white in (True, False):
+        seats = (True,) if args.no_color_swap else (True, False)
+        for candidate_is_white in seats:
             i = len(jobs)
             white_engine = candidate_engine if candidate_is_white else head_engine
             black_engine = head_engine if candidate_is_white else candidate_engine
@@ -982,6 +1122,7 @@ def main():
 
     candidate_points = 0.0
     wins = draws = losses = 0
+    aborted = 0
     sprt = Sprt(args.elo0, args.elo1, args.alpha, args.beta) if args.sprt else None
     sprt_decision = None
     candidate_stats = EngineStats()
@@ -1002,22 +1143,56 @@ def main():
         if args.sd is None and head_time != cand_time:
             assert ((white_time == cand_time) == candidate_is_white), \
                 "time-odds seat mapping inverted"
-        result_white, moves, move_evals, white_stats, black_stats = play_one_game(
-            white_engine, black_engine, f"{args.scratch}/g{i}", fen,
-            depth=args.sd,
-            white_time_sec=white_time, black_time_sec=black_time,
-            max_plies=args.max_plies, post=args.post,
-        )
+        try:
+            result_white, moves, move_evals, white_stats, black_stats, abort_info = (
+                play_one_game(
+                    white_engine, black_engine, f"{args.scratch}/g{i}", fen,
+                    depth=args.sd,
+                    white_time_sec=white_time, black_time_sec=black_time,
+                    max_plies=args.max_plies, post=args.post,
+                )
+            )
+        except (BrokenPipeError, OSError) as exc:
+            # One of the two engine subprocesses died mid-game (crashed,
+            # was OOM-killed, etc.) -- play_one_game's own abort_info
+            # machinery only covers READ-side failures (EOF/line-cap
+            # while scanning for a move); a WRITE to an already-dead
+            # process's stdin raises BrokenPipeError straight out of
+            # Engine._send instead, uncaught anywhere below this. Left
+            # unhandled, that exception propagates out of this worker
+            # thread's run() through the Future and kills the entire
+            # match (every other in-flight/queued game lost) over a
+            # single flaky engine process -- confirmed live against a
+            # polyglot/Stockfish side via stockfish_xboard. Treat it the
+            # same way as any other abort: no result guessed, game
+            # skipped, run continues.
+            result_white = None
+            moves, move_evals = [], []
+            white_stats, black_stats = EngineStats(), EngineStats()
+            abort_info = {
+                "reason": "engine-pipe-broken",
+                "exception": f"{type(exc).__name__}: {exc}",
+            }
         game_elapsed = time.monotonic() - game_start
-        candidate_result = result_white if candidate_is_white else (1.0 - result_white)
-        white_name = "candidate" if candidate_is_white else "baseline"
-        black_name = "baseline" if candidate_is_white else "candidate"
+        # result_white is None for an aborted game (play_one_game's mover
+        # failed to produce a legal move for a non-resign reason) -- do
+        # NOT guess a winner (see abort_info's own comment at the
+        # return sites in play_one_game for why this matters: it's
+        # exactly the bug that silently corrupted ~47% of an earlier
+        # self-play corpus). Propagate None through candidate_result too
+        # so run()'s caller can skip it rather than score it.
+        candidate_result = (
+            None if result_white is None else
+            result_white if candidate_is_white else (1.0 - result_white)
+        )
+        white_name = args.candidate_name if candidate_is_white else args.baseline_name
+        black_name = args.baseline_name if candidate_is_white else args.candidate_name
         game, _ = build_pgn_game(fen, moves, result_white, white_name, black_name,
                                   move_evals=move_evals)
         this_candidate_stats = white_stats if candidate_is_white else black_stats
         this_head_stats = black_stats if candidate_is_white else white_stats
         return (i, candidate_result, game, game_elapsed,
-                this_candidate_stats, this_head_stats)
+                this_candidate_stats, this_head_stats, abort_info)
 
     game_durations = []
     job_iter = iter(jobs)
@@ -1057,14 +1232,39 @@ def main():
                 finished, in_flight = wait(in_flight, return_when=FIRST_COMPLETED)
                 for fut in finished:
                     (i, candidate_result, game, game_elapsed,
-                     this_candidate_stats, this_head_stats) = fut.result()
+                     this_candidate_stats, this_head_stats, abort_info) = fut.result()
+                    print(game, file=pgn_out, end="\n\n")
+                    pgn_out.flush()
+
+                    if candidate_result is None:
+                        # Aborted game (play_one_game's mover failed to
+                        # produce a legal move for a non-resign reason --
+                        # see abort_info). Do NOT score it (no wins/
+                        # losses/draws/candidate_points/SPRT update, no
+                        # contribution to the `done` denominator used for
+                        # score/ETA) -- this is exactly the "silently
+                        # record a decisive result with no diagnostic
+                        # trail" bug that corrupted ~47% of an earlier
+                        # self-play corpus (docs/NNUE.md, 2026-10-17).
+                        # The game is still written to pgn_out above
+                        # (Result "*", which fit_k_and_label.py's
+                        # result_to_white_score() already treats as
+                        # "unknown, skip" for training) so it isn't lost
+                        # for post-mortem debugging, just not scored.
+                        aborted += 1
+                        log.write(f"game {i}: ABORTED (not scored) "
+                                  f"abort_info={abort_info} "
+                                  f"elapsed={game_elapsed:.1f}s\n")
+                        log.flush()
+                        print(f"  game {i}: ABORTED -- {abort_info}",
+                              file=sys.stderr)
+                        continue
+
                     done += 1
                     candidate_points += candidate_result
                     game_durations.append(game_elapsed)
                     candidate_stats.merge(this_candidate_stats)
                     head_stats.merge(this_head_stats)
-                    print(game, file=pgn_out, end="\n\n")
-                    pgn_out.flush()
                     if candidate_result == 1.0:
                         wins += 1
                     elif candidate_result == 0.0:
@@ -1131,7 +1331,7 @@ def main():
                   "(not waiting for in-flight games to finish naturally):",
                   file=sys.stderr)
             print_final_summary(done, candidate_points, wins, draws,
-                                 losses, candidate_stats, head_stats,
+                                 losses, aborted, candidate_stats, head_stats,
                                  sprt, sprt_decision, args)
             # os._exit, not sys.exit: skips the `with` block's normal
             # ThreadPoolExecutor teardown (wait=True by default -- would
@@ -1143,17 +1343,21 @@ def main():
             # the file objects' own __exit__.
             os._exit(130)
 
-    print_final_summary(done, candidate_points, wins, draws, losses,
+    print_final_summary(done, candidate_points, wins, draws, losses, aborted,
                          candidate_stats, head_stats, sprt, sprt_decision,
                          args)
 
 
-def print_final_summary(done, candidate_points, wins, draws, losses,
+def print_final_summary(done, candidate_points, wins, draws, losses, aborted,
                          candidate_stats, head_stats, sprt, sprt_decision,
                          args):
     """Shared by the normal end-of-match path and the ^C handler below --
     same CANDIDATE_SCORE/CANDIDATE_STATS/HEAD_STATS block either way, just
-    computed from whatever `done` games actually completed."""
+    computed from whatever `done` games actually completed. `aborted`
+    (not included in `done` or in any win/draw/loss/score arithmetic) is
+    the count of games where a mover failed to produce a legal move for
+    a non-resign reason -- see play_one_game's abort_info. Surfaced here
+    so an elevated count is visible instead of silently discarded."""
     n_games = done
     score = candidate_points / n_games if n_games else 0.5
     elo, err = elo_diff(score, n_games)
@@ -1182,10 +1386,11 @@ def print_final_summary(done, candidate_points, wins, draws, losses,
                   f"cand={args.cand_time}s)")
     else:
         tc_str = f" TC=st{args.head_time}"
+    aborted_str = f" ABORTED={aborted}" if aborted else ""
     print(f"CANDIDATE_SCORE={score:.4f} GAMES={n_games} "
           f"WINS={wins} DRAWS={draws} LOSSES={losses} "
           f"ELO={elo:+.1f}+/-{err:.1f} LOWER95={lower95:.4f}"
-          f"{tc_str}{sprt_str}")
+          f"{tc_str}{sprt_str}{aborted_str}")
     print(f"CANDIDATE_STATS: {candidate_stats.summary()}")
     print(f"HEAD_STATS:      {head_stats.summary()}")