diff options
| author | Scott Gasch <[email protected]> | 2026-10-20 09:26:44 -0700 |
|---|---|---|
| committer | Scott Gasch <[email protected]> | 2026-10-20 09:26:44 -0700 |
| commit | c8c6bbc2c1a103e5b07b079a0f8bdf66c7724d80 (patch) | |
| tree | b6c2137c9ac577fa83b1e948fa0311d53dfe445d /src/eval_tune | |
| parent | 3b5d3677bb14ae14728a1c4b221176bb83667d92 (diff) | |
Wire NNUE eval into search, build the training pipeline, and add a fast
move/eval-agreement benchmark against Stockfish (tests/lop)
Implements the architecture decisions in src/docs/NNUE.md: HalfKAv2_hm
feature set, int16/int8 quantization, per-ply accumulator backup for
king moves, and a lazy-eval-ready integration into search.c/eval.c.
Adds the full annotate_pgns/ training pipeline (extraction, K-fit/
labeling, quiet filtering, tensor building, training, export) plus an
end-to-end orchestrator (run_nnue_pipeline.py) and validation tooling
(sanity_check_net.py, compare_vs_stockfish_net.py). Adds tests/lop:
a 10k-position list-of-positions benchmark comparing engine eval rank
and move choice against pre-computed Stockfish/lc0 references, much
cheaper than a full self-play match for day-to-day progress checks.
First sd10-matched self-play result against classic HCE: LLR +2.28
(H1=+2.94), -72=119+169 over 360 games, confirming the net is
"smarter per node" at matched depth. A real-time (--st 4) match is
currently weaker (candidate loses ~3 plies of depth to a ~3.3-3.7x
nps gap against an unrecalibrated search), consistent with
NNUE.md's adoption-criterion section -- pruning/reduction constants
still need retuning for the new eval's score scale before a fair
real-time comparison is possible.
Co-Authored-By: Claude Sonnet 5 <[email protected]>
Diffstat (limited to 'src/eval_tune')
| -rw-r--r-- | src/eval_tune/build_cross_sectional_all.py | 228 | ||||
| -rw-r--r-- | src/eval_tune/compare_vs_crafty.py | 597 | ||||
| -rwxr-xr-x | src/eval_tune/match_play.py | 275 | ||||
| -rwxr-xr-x | src/eval_tune/nnue_vs_head.sh | 219 | ||||
| -rw-r--r-- | src/eval_tune/relabel_selfplay_pgn.py | 112 | ||||
| -rwxr-xr-x | src/eval_tune/stockfish_training_corpus.sh | 13 | ||||
| -rwxr-xr-x | src/eval_tune/test_vs_head.sh | 94 | ||||
| -rwxr-xr-x | src/eval_tune/typhoon_hce_training_corpus.sh | 173 | ||||
| -rwxr-xr-x | src/eval_tune/typhoon_nnue_training_corpus.sh | 224 |
9 files changed, 1884 insertions, 51 deletions
diff --git a/src/eval_tune/build_cross_sectional_all.py b/src/eval_tune/build_cross_sectional_all.py new file mode 100644 index 0000000..144c87c --- /dev/null +++ b/src/eval_tune/build_cross_sectional_all.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Build a larger cross-sectional position pool (successor to +tests/cross_sectional_200k.ep_) out of the full twic.pgn + +lichess_elite*.pgn corpus. + +Unlike sample_eval_calibration_positions.py's random-byte-offset +sampling (one position per game, uniform-random ply), this does an +exhaustive streaming pass over every game and pulls 5 positions per +qualifying game at fixed fractional points in the game +(early/early-mid/mid/mid-late/late) -- Scott's own framing. Fractional- +ply sampling structurally balances game-stage representation (every +kept game contributes exactly one position to each of the 5 stage +buckets), which random-ply sampling does not. + +Deliberately does NOT filter for quiescence -- these positions are +`go` roots for the engine's own search (which resolves tactics via its +own qsearch at every node), not Texel-tuning eval-only positions. See +sample_eval_calibration_positions.py's own docstring for the same +reasoning: a cross-sectional pool should look like "all positions a +real search tree touches", not just the quiet subset. + +Ply fractions: 5 samples come from 6 equal segments (1/6..5/6 of the +game's ply count), not literal 1/5..5/5 -- 5/5 is the final move of +the game, frequently checkmate/stalemate/resignation-adjacent, which +`go` can't search meaningfully. Interior boundaries only. +""" +import argparse +import collections +import io +import sys +from pathlib import Path + +import chess +import chess.pgn + +STAGE_NAMES = ["early", "early_mid", "mid", "mid_late", "late"] +FRACTIONS = [1 / 6, 2 / 6, 3 / 6, 4 / 6, 5 / 6] + +PIECE_VALUES = {chess.KNIGHT: 3, chess.BISHOP: 3, chess.ROOK: 5, chess.QUEEN: 9} + + +def material_bucket(board): + """Same bucketing as eval.c's _MaterialBucket / the calibration + script's own copy: combined non-pawn army value (pawn units) / 8, + capped at 7 (0 = bare kings, 7 = full army).""" + total = 0 + for pt, v in PIECE_VALUES.items(): + total += v * len(board.pieces(pt, chess.WHITE)) + total += v * len(board.pieces(pt, chess.BLACK)) + return min(7, total // 8) + + +def iter_game_texts(pgn_path): + """Stream raw PGN text blocks, one game at a time, without loading + the whole file -- these corpora run into the GB range (twic.pgn + alone is 3.5GB).""" + with open(pgn_path, "r", encoding="utf-8", errors="replace") as f: + header_lines = [] + movetext_lines = [] + in_headers = True + for line in f: + if line.startswith("["): + if not in_headers and (header_lines or movetext_lines): + yield "".join(header_lines) + "\n" + "".join(movetext_lines) + header_lines, movetext_lines = [], [] + header_lines.append(line) + in_headers = True + elif line.strip() == "": + if header_lines and not movetext_lines: + in_headers = False + else: + in_headers = False + movetext_lines.append(line) + if header_lines or movetext_lines: + yield "".join(header_lines) + "\n" + "".join(movetext_lines) + + +def normalized_fen_key(fen): + """Board+side-to-move+castling+ep, dropping halfmove/fullmove + counters -- used only for dedup accounting, never written out.""" + parts = fen.split() + return " ".join(parts[:4]) + + +def process_pgn(path, min_elo, min_ply, stats, seen_keys, out_f): + kept_games = 0 + scanned_games = 0 + for raw in iter_game_texts(path): + scanned_games += 1 + game = chess.pgn.read_game(io.StringIO(raw)) + if game is None: + continue + headers = game.headers + try: + we = int(headers.get("WhiteElo", "0")) + be = int(headers.get("BlackElo", "0")) + except ValueError: + we = be = 0 + if min_elo and (we < min_elo or be < min_elo): + stats["rejected_elo"] += 1 + continue + + moves = list(game.mainline_moves()) + if len(moves) < min_ply: + stats["rejected_short"] += 1 + continue + + result = headers.get("Result", "*") + stats["games_by_result"][result] += 1 + stats["source_games"][path.name] += 1 + kept_games += 1 + + board = game.board() + move_idx = 0 + boundaries = [round(f * len(moves)) for f in FRACTIONS] + # de-dup consecutive identical boundaries (very short-but-kept + # games can round two fractions to the same ply) + seen_boundaries = set() + bi = 0 + for target_ply in boundaries: + target_ply = max(1, min(target_ply, len(moves) - 1)) + while move_idx < target_ply: + board.push(moves[move_idx]) + move_idx += 1 + stage = STAGE_NAMES[bi] + bi += 1 + if target_ply in seen_boundaries: + stats["skipped_duplicate_boundary"] += 1 + continue + seen_boundaries.add(target_ply) + if board.is_game_over(): + stats["skipped_game_over"] += 1 + continue + fen = board.fen() + key = normalized_fen_key(fen) + stats["stage_total"][stage] += 1 + stats["material_bucket"][material_bucket(board)] += 1 + stats["side_to_move"]["white" if board.turn else "black"] += 1 + if key in seen_keys: + stats["duplicate_fens"] += 1 + stats["stage_duplicates"][stage] += 1 + continue + seen_keys.add(key) + out_f.write(f"setboard {fen}\n") + out_f.write("go\n") + stats["written"] += 1 + stats["stage_written"][stage] += 1 + + stats["scanned_games"] += scanned_games + stats["kept_games"] += kept_games + return kept_games + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("pgn_paths", nargs="+") + ap.add_argument("--min-elo", type=int, default=2000) + ap.add_argument("--min-ply", type=int, default=30, + help="minimum game length in plies to be sampled " + "at all -- filters short draws/miniatures") + ap.add_argument("--out", required=True) + ap.add_argument("--progress-every", type=int, default=50000) + args = ap.parse_args() + + stats = { + "scanned_games": 0, + "kept_games": 0, + "rejected_elo": 0, + "rejected_short": 0, + "skipped_game_over": 0, + "skipped_duplicate_boundary": 0, + "duplicate_fens": 0, + "written": 0, + "games_by_result": collections.Counter(), + "source_games": collections.Counter(), + "stage_total": collections.Counter(), + "stage_written": collections.Counter(), + "stage_duplicates": collections.Counter(), + "material_bucket": collections.Counter(), + "side_to_move": collections.Counter(), + } + seen_keys = set() + + with open(args.out, "w") as out_f: + for p in args.pgn_paths: + path = Path(p) + print(f"processing {path.name}...", file=sys.stderr) + before = stats["kept_games"] + process_pgn(path, args.min_elo, args.min_ply, stats, seen_keys, out_f) + print(f" kept {stats['kept_games'] - before} games " + f"(running total written: {stats['written']})", file=sys.stderr) + + print("\n=== summary ===") + print(f"games scanned: {stats['scanned_games']:,}") + print(f"games kept: {stats['kept_games']:,}") + print(f" rejected (elo): {stats['rejected_elo']:,}") + print(f" rejected (too short): {stats['rejected_short']:,}") + print(f"positions written: {stats['written']:,}") + print(f"duplicate fens skipped: {stats['duplicate_fens']:,} " + f"({100.0*stats['duplicate_fens']/max(1,stats['written']+stats['duplicate_fens']):.1f}%)") + print(f"game-over boundaries skipped: {stats['skipped_game_over']:,}") + print() + print("by result:") + for r, c in stats["games_by_result"].most_common(): + print(f" {r:>8}: {c:,}") + print() + print("by source file:") + for s, c in stats["source_games"].most_common(): + print(f" {s}: {c:,}") + print() + print("by game stage (written / total-before-dedup):") + for stage in STAGE_NAMES: + w = stats["stage_written"][stage] + t = stats["stage_total"][stage] + d = stats["stage_duplicates"][stage] + print(f" {stage:>10}: written={w:,} dup_rate={100.0*d/max(1,t):.1f}%") + print() + print("by material bucket (0=bare kings .. 7=full army):") + for b in range(8): + print(f" bucket {b}: {stats['material_bucket'][b]:,}") + print() + print("by side to move:") + for k, c in stats["side_to_move"].items(): + print(f" {k}: {c:,}") + + +if __name__ == "__main__": + main() diff --git a/src/eval_tune/compare_vs_crafty.py b/src/eval_tune/compare_vs_crafty.py new file mode 100644 index 0000000..318ff87 --- /dev/null +++ b/src/eval_tune/compare_vs_crafty.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python3 +""" +Cross-engine static-eval audit: compare typhoon's `eval` command output +against a *crowd* of other hand-crafted-eval engines (Crafty, Kiwi, Fruit, +GNU Chess -- all pre-NNUE, additive-term evals, so genuinely comparable to +typhoon's own, unlike Stockfish's NNUE), rank the positions where typhoon +is the outlier against the crowd's median, and attribute the gap back to +typhoon's own named EVAL_TERM lines (from a build with EVAL_DUMP=1). + +Methodology, and the gotchas that cost real time to find (2026-09-2x +eval audit session): + + - Build typhoon with EVAL_DUMP=1 (see CLAUDE.md's build instructions) + to get the per-term breakdown table, not just the "Static eval:" + total. The breakdown itself is a SEPARATE command, `dump eval` + (EvalTraceReport() in testeval.c) -- it reports on the terms + recorded by the *last* Eval() call, so the script sends `eval` then + `dump eval` back to back for every position, not `eval` alone. + + - Run typhoon with --hash none. Not an optimization -- a correctness- + adjacent perf fix. SetBoardCommand -> PreGameReset(FALSE) -> + ClearHashTable() unconditionally, so every single `setboard` with a + real --hash size does a full memset of the whole hash table. Over + 2000 positions this was the entire difference between ~46s and + ~2.2s (21x) for byte-identical output. Always pass --hash none for + eval-only batch runs; there is no search happening, so there is + nothing to hash. + + - Feed positions via `script <file>` (one `setboard <fen>` / `eval` + pair per line), not a single giant --command string. A few thousand + positions' worth of inline command text can exceed the shell's/ + kernel's argv length limits; `script` reads from a file instead and + has no such ceiling. + + - Only ONE process per engine for the whole batch -- every engine here + correctly returns different, real scores across repeated + setboard/eval calls in a single session; none needs restarting per + position. (Verified directly this session by feeding + /usr/local/bin/crafty three visibly different positions in one + stdin session and getting three different totals.) + + - Use /usr/local/bin/crafty (whichever `crafty` resolves to on PATH), + NOT ~/programming/chess/crafty-25.2/crafty (formerly ~/crafty/crafty + -- that whole directory was moved to this path). The latter is a + different, older build (confirmed via md5) that a sibling script + here (compare_king_safety_crafty.py) found DOES silently reuse the + first position's cached score on a second setboard+score in one + session -- that bug is specific to that binary, not a general + Crafty limitation, and does not reproduce against + /usr/local/bin/crafty. If you ever see suspiciously identical + Crafty totals across different positions, check which crafty + binary you're actually calling before assuming a new engine bug. + + - Crowd engines each have their own sign convention for the "eval" + command's printed number, confirmed empirically (setboard a position + with white down a whole knight and check the sign/magnitude), not + assumed from reading their source alone: + * crafty `score`, kiwi `eval` -- print the score from WHITE's + perspective already, no flip needed. + * fruit `eval` -- prints from the SIDE-TO-MOVE's perspective + (negamax convention: eval.cpp's `eval()` explicitly negates for + Black at the very end). Must flip sign when it's Black to move + to get a White-perspective number comparable to the others. + * gnuchess (gnuchessr) `score` -- ALWAYS prints "(white to + move)" and an always-White-relative number, regardless of the + loaded position's actual active color. Confirmed empirically: + loading the identical position via two xpos files differing + only in trailing "w"/"b" produced byte-identical output both + times. Root cause (read, not guessed): `xget` -> GetXGame() + (dspcom.c) sets the `computer` global from the file's color + field but never touches the separate `player` global that + ScorePosition()'s perspective and the printed "(%s to move)" + label actually key off -- `player` is a zero-initialized global + that only search.c's search loop ever updates, so outside of an + actual search it just stays at its BSS-zero value (White). Net + effect: harmless for this script (means no stm-relative flip is + ever needed for gnuchess) but it DOES mean gnuchess's score + never reflects tempo/mobility-from-whoever's-actually-on-move + nuance -- a known accuracy ceiling on this crowd member, not a + bug in this script. + Getting a sign convention wrong silently halves the apparent + disagreement rate (a sign-flipped engine agrees with the crowd + exactly when it should maximally disagree, and vice versa) rather + than crashing, so it does not announce itself -- always spot check + a lopsided material position per engine before trusting a new one's + numbers. + + - Kiwi's own `kiwi` binary in ~/programming/chess/kiwi can be stale + relative to its own source (confirmed: mtimes showed the checked-in + binary predated winboard.cxx/engine.cxx, and it silently mis-parses + `eval` as an illegal `usermove` rather than erroring, because + command-table dispatch on prefix match falls through to + cmd_OpponentMoves on a lookup miss) -- there was already a correctly + rebuilt `kiwi.new` sitting alongside it in that same session, which + is what this script actually points at + (KIWI_BIN, below). If kiwi's numbers ever look wrong, rebuild with + `env OBJDIR=. LD=clang++ CC=clang++ gmake` from ~/programming/chess/kiwi + (its Makefile uses GNU-make-only `$(addprefix ...)`, so it must be + `gmake`, not BSD `make`; g++ doesn't exist on this box, hence the + clang++ overrides) and re-point KIWI_BIN at the fresh binary. + + - gnuchess (gnuchessr, ~/programming/chess/gnuchess/src) has no + `setboard`/`position fen` command at all -- the only way to load an + arbitrary position is `xget`, an interactive prompt that reads a + FILE in gnuchess's old xboard.position.read format (just FEN's + piece-placement and active-color fields, one line, no + castling/ep/clock -- castling rights are simply lost, an inherent + approximation for this crowd member). `_build_gnuchess_script` + writes one such tiny file per position under + /tmp/typhoon/eval_audit/gnuchess/ rather than trying to reuse one + filename, since the position must be fully written before the + single stdin-scripted process reaches that `xget` line. Also: the + process needs one leading blank line to satisfy its startup + "Enter [moves] minutes[:sec]..." time-control prompt before the + "Your move is?" loop (where `xget`/`score`/`quit` are valid + commands) even begins reading commands -- a script missing that + blank line gets its first real command silently swallowed by the + time-control prompt instead. Also needs its OWN directory as cwd + (same gotcha class as Glaurung/kpk.bin in an earlier version of + this crowd): run from elsewhere it prints "NO LANGFILE (file + gnuchess.lang not found)" and exits immediately instead of falling + back gracefully. + + - Compare typhoon against the crowd's MEDIAN, not vs. any single + other engine and not vs. the mean. Median is robust to any one + crowd engine being the actual outlier on a given position (e.g. a + known-narrow specialization one of them has and the others don't); + a mean lets one bad engine drag the "consensus" toward itself. + Historically (see the old Crafty-only version of this script) + averaging in an NNUE engine (Stockfish) made typhoon look like the + outlier for tracking a hand-crafted eval correctly (9 of 10 checked + outliers were actually typhoon+Crafty agreeing within ~0.3 pawns + with Stockfish alone ~12-15 pawns off) -- same failure mode extends + to any structurally-different evaluator diluting the crowd, so keep + the crowd restricted to same-family (additive, hand-tuned) evals. + + - This box's kern.ipc.semmni (max SysV semaphore sets) is only 50. + Every typhoon process leaks its semaphore set on `kill -9` (a clean + `quit`/EOF releases it; a hard kill does not, regardless of the + unix.c pthread-mutex/condvar rewrite -- that fixed a different + leak). Repeatedly killing hung/timed-out typhoon processes during + debugging can silently exhaust the limit; once exhausted, EVERY new + typhoon process falls back to a ~1000x-slower 100ms polling loop + for input, which looks exactly like a hang or a new performance + regression. Check with `ipcs -s | grep $(whoami) | wc -l` before + concluding a slowdown is a code bug; clean up with `ipcrm -s <id>` + for each leaked entry (or `for id in $(ipcs -s | grep $(whoami) | + awk '{print $2}'); do ipcrm -s "$id"; done`). + +Usage: + + python3 compare_vs_crafty.py <fen_source> [--limit N] [--top K] + [--engines crafty,kiwi,fruit,gnuchess] + + <fen_source> is either a path to an .ep_-style file (lines of + "setboard <fen>" / "go", as in ../tests/*.ep_) or a plain file of one + FEN per line. Requires a typhoon binary built with EVAL_DUMP=1 at + ../typhoon (build it first: `gmake clean && gmake -j5 GENETIC=1 + PERF_COUNTERS=1 MP=1 SIXTYFOUR=1 EVAL_DUMP=1` from src/), plus + whichever crowd engine binaries are selected (all on by default; + see ENGINE_SPECS for their expected paths). + +Output: prints the top-K disagreements (|typhoon(white) - +crowd_median(white)|), each crowd engine's own white-perspective score +for context, and typhoon's largest contributing EVAL_TERM line(s) from +its own perspective -- a quick first read on which term is driving the +gap, before digging in by hand. +""" +import argparse +import re +import statistics +import subprocess +import sys +from pathlib import Path + +SRC_DIR = Path(__file__).resolve().parent.parent +TYPHOON_BIN = SRC_DIR / "typhoon" +SCRATCH = Path("/tmp/typhoon/eval_audit") + +CHESS_DIR = Path.home() / "programming" / "chess" + +# Every crowd engine speaks a different protocol/command for "set this +# position, print static eval." `kind` selects how the position+eval +# commands are framed; `parse_re` must capture exactly the numeric score +# (possibly negative) as group 1; `scale` converts its native units to +# pawns; `stm_relative` marks engines whose printed number is from the +# SIDE TO MOVE's perspective (needs a sign flip on Black-to-move +# positions) rather than already being White-relative. +ENGINE_SPECS = { + "crafty": dict( + binary="crafty", cwd=None, kind="xboard", + startup="ponder off\n", pos_fmt="setboard {fen}\n", eval_cmd="score", + parse_re=r'^total\.+\s+([+-]?\d+\.\d+)', scale=1.0, stm_relative=False, + ), + "kiwi": dict( + binary=str(CHESS_DIR / "kiwi" / "kiwi.new"), cwd=None, kind="xboard", + startup="", pos_fmt="setboard {fen}\n", eval_cmd="eval", + parse_re=r'^Static eval\s*=\s*([+-]?\d+)', scale=0.01, stm_relative=False, + ), + "fruit": dict( + binary=str(CHESS_DIR / "fruit" / "fruit"), cwd=None, kind="uci", + startup="", pos_fmt="position fen {fen}\n", eval_cmd="eval", + parse_re=r'info string eval\s+([+-]?\d+)', scale=0.01, stm_relative=True, + ), + "gnuchess": dict( + binary=str(CHESS_DIR / "gnuchess" / "src" / "gnuchessr"), + cwd=str(CHESS_DIR / "gnuchess" / "src"), + kind="gnuchess", scale=0.01, stm_relative=False, + # Not anchored at line-start: gnuchess's "Your move is? " prompt + # precedes "Static score..." on the SAME line, unlike every + # other engine here which prints its own line cleanly. + parse_re=r'Static score \(\w+ to move\):\s*([+-]?\d+)', + ), +} + + +def extract_fens(path: Path) -> list[str]: + fens = [] + for line in path.read_text().splitlines(): + line = line.strip() + if line.startswith("setboard "): + fens.append(line[len("setboard "):].strip()) + elif line and not line.startswith(("go", "#")) and " " in line and "/" in line: + # plain FEN file + fens.append(line) + return fens + + +def run_typhoon_eval_dump(fens: list[str]) -> str: + SCRATCH.mkdir(parents=True, exist_ok=True) + script_path = SCRATCH / "typhoon_script.txt" + out_path = SCRATCH / "typhoon_out.txt" + with open(script_path, "w") as f: + for fen in fens: + # `eval` alone only prints the "Static eval:" total; the + # per-term "--- WHITE ... BLACK" breakdown table comes from + # the separate `dump eval` command (EvalTraceReport(), + # testeval.c), which reports on the terms from the *last* + # Eval() call -- so it must follow `eval` for the same + # position, not stand alone. + f.write(f"setboard {fen}\neval\ndump eval\n") + cmd = [str(TYPHOON_BIN), "--cpus", "1", "--hash", "none", + "--logfile", "-", "--batch", "--command", + f"force; book name /nonexistent.book.bin; script {script_path}"] + with open(out_path, "w") as out: + subprocess.run(cmd, stdout=out, stderr=subprocess.STDOUT, cwd=SRC_DIR, + timeout=max(60, len(fens) // 20)) + return out_path.read_text() + + +def run_crowd_engine(name: str, fens: list[str]) -> str: + spec = ENGINE_SPECS[name] + if spec["kind"] == "gnuchess": + script = _build_gnuchess_script(fens) + else: + script = spec["startup"] + for fen in fens: + script += spec["pos_fmt"].format(fen=fen) + spec["eval_cmd"] + "\n" + script += "quit\n" + p = subprocess.run([spec["binary"]], input=script, capture_output=True, + text=True, cwd=spec["cwd"], timeout=max(60, len(fens) // 20)) + return p.stdout + + +def _build_gnuchess_script(fens: list[str]) -> str: + """gnuchess (gnuchessr) has no `setboard`/`position fen` -- the only + way to load an arbitrary position is `xget`, which prompts for a + filename and reads a file in its own old xboard.position.read + format: just FEN's first two fields (piece placement + active + color), one per file, no castling/ep/clock info. Also: a leading + blank line is required first to accept the default time control + prompt before the "Your move is?" command loop (where `xget` is a + valid command) even starts accepting input.""" + gc_dir = SCRATCH / "gnuchess" + gc_dir.mkdir(parents=True, exist_ok=True) + script = "\n" + for i, fen in enumerate(fens): + parts = fen.split() + pos_path = gc_dir / f"pos_{i}.xpos" + pos_path.write_text(f"{parts[0]} {parts[1]}\n") + script += f"xget\n{pos_path}\nscore\n" + script += "quit\n" + return script + + +def parse_typhoon(text: str, n: int): + evals = [float(x) for x in + re.findall(r'^Static eval:\s*([+-]?\d+\.\d+)', text, re.M)] + assert len(evals) == n, f"expected {n} typhoon evals, got {len(evals)}" + return evals + + +def parse_crowd_engine(name: str, text: str, stm: list[str], n: int): + """Returns n scores in pawns, from WHITE's perspective.""" + spec = ENGINE_SPECS[name] + raw = [float(x) for x in re.findall(spec["parse_re"], text, re.M)] + assert len(raw) == n, ( + f"expected {n} {name} evals, got {len(raw)} -- protocol/parse_re " + f"mismatch, or {name}'s binary crashed/changed output format") + white = [] + for val, side in zip(raw, stm): + v = val * spec["scale"] + if spec["stm_relative"] and side == 'b': + v = -v + white.append(v) + return white + + +LABEL_RE_CACHE = {} + + +def term_labels_from_eval_c() -> list[str]: + """Pull every EVAL_TERM(...) call's label string directly out of + eval.c, so this stays in sync with the source without needing a + separately-maintained list.""" + src = (SRC_DIR / "eval.c").read_text() + labels = set() + for m in re.finditer(r'EVAL_TERM\(', src): + start = m.end() + depth = 1 + i = start + while depth > 0 and i < len(src): + if src[i] == '(': + depth += 1 + elif src[i] == ')': + depth -= 1 + i += 1 + # Allow escaped quotes/backslashes inside the string literal + # (e.g. eval.c's own "in/out of \"action\"" label) -- a naive + # [^"]* stops at the first escaped quote and can pick up a + # spurious empty trailing match instead of the real label. + strs = re.findall(r'"((?:[^"\\]|\\.)*)"', src[start:i - 1]) + if strs: + labels.add(strs[-1].replace('\\"', '"')) + return sorted(labels, key=len, reverse=True) + + +def get_label_re(): + if 're' not in LABEL_RE_CACHE: + labels = term_labels_from_eval_c() + LABEL_RE_CACHE['re'] = re.compile( + '(' + '|'.join(re.escape(l) for l in labels) + r')\s*(-|[a-h][1-8])?\s+(-?\d+)') + return LABEL_RE_CACHE['re'] + + +def parse_terms_for_block(block: str): + """block is the text between '--- WHITE' and 'Static eval:' for one + position. Returns {label: contribution_to_WHITE} (white/black columns + split at column 40, matching the dump's fixed layout). Always + White-relative regardless of side to move -- each column is that + color's own raw contribution, so White's column adds and Black's + subtracts from the White-relative total, independent of stm. (Do not + key the sign off stm: that gives the contribution to the side TO + MOVE instead, which flips sign on every Black-to-move position and + silently destroys any correlation against a White-relative residual.) + """ + label_re = get_label_re() + contrib = {} + for line in block.splitlines(): + for half, color in ((line[:40], 'w'), (line[40:], 'b')): + for m in label_re.finditer(half): + label, val = m.group(1), int(m.group(3)) + sign = 1 if color == 'w' else -1 + contrib[label] = contrib.get(label, 0) + sign * val + return contrib + + +def split_typhoon_blocks(text: str) -> list[str]: + # Each block runs from its own "--- WHITE" header up to (but not + # including) the NEXT position's leading board-dump text -- trim at + # this block's own "Static eval:" line, or the raw split would + # include the following position's FEN/hash/board-art/"Material:" + # preamble as trailing garbage, which can spuriously match the term + # regex (large hex-signature/material digit runs sitting near label + # text by coincidence). + parts = re.split(r'(?=^--- WHITE)', text, flags=re.M) + blocks = [] + for p in parts: + if not p.startswith('--- WHITE'): + continue + m = re.search(r'^Static eval:.*$', p, flags=re.M) + blocks.append(p[:m.end()] if m else p) + return blocks + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("fen_source") + ap.add_argument("--limit", type=int, default=2000, + help="max positions to sample (default 2000)") + ap.add_argument("--top", type=int, default=20, + help="how many biggest disagreements to print") + ap.add_argument("--engines", default=",".join(ENGINE_SPECS), + help=f"comma-separated crowd engines to use, from " + f"{{{','.join(ENGINE_SPECS)}}} (default: all)") + ap.add_argument("--sign-deadband", type=float, default=0.5, + help="pawns; both |typhoon| and |crowd median| (or " + "unanimous crowd value) must clear this to count " + "as a real sign disagreement, not ~0 noise " + "(default 0.5)") + ap.add_argument("--spread-floor", type=float, default=0.25, + help="pawns; floor on crowd stdev used when ranking " + "gap/spread so a near-unanimous crowd doesn't " + "produce a division-by-near-zero blowup " + "(default 0.25)") + args = ap.parse_args() + + engines = [e.strip() for e in args.engines.split(",") if e.strip()] + for e in engines: + if e not in ENGINE_SPECS: + sys.exit(f"unknown engine {e!r} -- choose from {list(ENGINE_SPECS)}") + if not Path(ENGINE_SPECS[e]["binary"]).is_absolute() and \ + ENGINE_SPECS[e]["cwd"] is None: + continue # resolved via PATH (e.g. crafty) + if not TYPHOON_BIN.exists(): + sys.exit(f"missing {TYPHOON_BIN} -- build with EVAL_DUMP=1 first " + "(see this file's docstring)") + + fens = extract_fens(Path(args.fen_source))[:args.limit] + stm = [f.split()[1] for f in fens] + print(f"{len(fens)} positions, crowd = {engines}", file=sys.stderr) + + ty_text = run_typhoon_eval_dump(fens) + ty_evals = parse_typhoon(ty_text, len(fens)) + ty_white = [e if s == 'w' else -e for e, s in zip(ty_evals, stm)] + + crowd_white = {} + for name in engines: + text = run_crowd_engine(name, fens) + crowd_white[name] = parse_crowd_engine(name, text, stm, len(fens)) + + blocks = split_typhoon_blocks(ty_text) + assert len(blocks) == len(fens), ( + f"{len(blocks)} eval-dump blocks vs {len(fens)} positions -- " + "did the typhoon binary actually have EVAL_DUMP=1?") + contribs = [parse_terms_for_block(b) for b in blocks] + + n = len(fens) + per_pos = [] # (median, stdev, residual, is_extreme, extreme_dir) + for i in range(n): + crowd_vals = [crowd_white[name][i] for name in engines] + median = statistics.median(crowd_vals) + stdev = statistics.pstdev(crowd_vals) if len(crowd_vals) > 1 else 0.0 + residual = ty_white[i] - median + all_vals = crowd_vals + [ty_white[i]] + is_max = ty_white[i] == max(all_vals) and all_vals.count(ty_white[i]) == 1 + is_min = ty_white[i] == min(all_vals) and all_vals.count(ty_white[i]) == 1 + per_pos.append((median, stdev, residual, is_max, is_min)) + + def print_position(i, extra=""): + median = per_pos[i][0] + crowd_str = " ".join(f"{name}={crowd_white[name][i]:+6.2f}" + for name in engines) + print(f"{extra}ty(w)={ty_white[i]:+6.2f} median(w)={median:+6.2f} " + f"{crowd_str} {fens[i]}") + top_terms = sorted(contribs[i].items(), key=lambda kv: -abs(kv[1]))[:4] + for label, val in top_terms: + print(f" {label:38s} {val:+5d}") + + # --- 1. Raw magnitude disagreements (original behavior) ------------- + diffs = sorted(range(n), key=lambda i: -abs(per_pos[i][2])) + print(f"\n=== Top {args.top} disagreements " + f"(|typhoon(white) - crowd_median(white)|) ===\n") + for i in diffs[:args.top]: + print_position(i, extra=f"diff={per_pos[i][2]:+7.2f} ") + + # --- 2. Gap normalized by crowd spread ------------------------------- + # A raw pawn gap conflates "typhoon is wrong" with "this position is + # just wild and the whole crowd disagrees with itself." Dividing by + # the crowd's own stdev (floored, so a near-unanimous crowd doesn't + # blow up the ratio) surfaces cases where typhoon breaks from an + # otherwise-tight consensus, even by a modest raw amount. + spread_ranked = sorted( + range(n), key=lambda i: -abs(per_pos[i][2]) / max(per_pos[i][1], args.spread_floor)) + print(f"\n=== Top {args.top} disagreements normalized by crowd spread " + f"(|residual| / max(crowd_stdev, {args.spread_floor})) ===\n") + for i in spread_ranked[:args.top]: + ratio = abs(per_pos[i][2]) / max(per_pos[i][1], args.spread_floor) + print_position(i, extra=f"ratio={ratio:6.2f} stdev={per_pos[i][1]:5.2f} ") + + # --- 3. Typhoon as the lone extreme value ---------------------------- + extremes = [i for i in range(n) if per_pos[i][3] or per_pos[i][4]] + extremes.sort(key=lambda i: -abs(per_pos[i][2])) + print(f"\n=== Positions where typhoon is the strict min or max of " + f"typhoon+crowd ({len(extremes)} of {n}) ===\n") + for i in extremes[:args.top]: + direction = "HIGHEST" if per_pos[i][3] else "LOWEST" + print_position(i, extra=f"typhoon is {direction} ") + + # --- 4. Sign disagreements ------------------------------------------- + db = args.sign_deadband + vs_median, vs_unanimous = [], [] + for i in range(n): + ty, median = ty_white[i], per_pos[i][0] + if abs(ty) >= db and abs(median) >= db and (ty > 0) != (median > 0): + vs_median.append(i) + crowd_vals = [crowd_white[name][i] for name in engines] + if all(abs(v) >= db for v in crowd_vals) and \ + len({v > 0 for v in crowd_vals}) == 1: + crowd_sign_positive = crowd_vals[0] > 0 + if abs(ty) >= db and (ty > 0) != crowd_sign_positive: + vs_unanimous.append(i) + print(f"\n=== Typhoon disagrees with crowd MEDIAN's sign " + f"(both sides >= {db} pawns) -- {len(vs_median)} positions ===\n") + for i in vs_median[:args.top]: + print_position(i) + print(f"\n=== Typhoon disagrees with a UNANIMOUS crowd's sign " + f"(every engine agrees, typhoon alone doesn't; >= {db} pawns) " + f"-- {len(vs_unanimous)} positions ===\n") + for i in vs_unanimous[:args.top]: + print_position(i) + if "gnuchess" in engines and len(engines) > 1: + others = [e for e in engines if e != "gnuchess"] + vs_unanimous_no_gc = [] + for i in range(n): + vals = [crowd_white[name][i] for name in others] + if all(abs(v) >= db for v in vals) and len({v > 0 for v in vals}) == 1: + if abs(ty_white[i]) >= db and (ty_white[i] > 0) != (vals[0] > 0): + vs_unanimous_no_gc.append(i) + print(f"\n (gnuchess drops castling rights and ignores stm; " + f"same check EXCLUDING gnuchess: " + f"{len(vs_unanimous_no_gc)} positions)") + + # --- 5. Per-term correlation with the White-relative residual -------- + # For each EVAL_TERM label, correlate its own signed White-relative + # contribution against ty_white - crowd_median across the whole + # sample. A term whose contribution consistently tracks the residual + # is a candidate for being over- or under-weighted relative to the + # crowd's implicit consensus; a term with no correlation isn't + # driving the disagreement even if it's large in any one position. + residuals = [p[2] for p in per_pos] + all_labels = set() + for c in contribs: + all_labels.update(c.keys()) + term_stats = [] + for label in all_labels: + vals = [contribs[i].get(label, 0) for i in range(n)] + if len(set(vals)) < 2: + continue # constant (e.g. always zero) -- correlation undefined + try: + corr = statistics.correlation(vals, residuals) + except statistics.StatisticsError: + continue + mean_abs = sum(abs(v) for v in vals) / n + term_stats.append((label, corr, mean_abs, corr * mean_abs)) + term_stats.sort(key=lambda t: -abs(t[3])) + print(f"\n=== EVAL_TERM correlation with (typhoon - crowd_median), " + f"whole sample (n={n}) ===") + print("(positive corr = term grows typhoon's score in the same " + "direction as its disagreement with the crowd -- a candidate " + "for being over-weighted; negative = under-weighted)\n") + for label, corr, mean_abs, score in term_stats[:20]: + print(f" {label:38s} corr={corr:+.3f} mean|contrib|={mean_abs:6.2f} " + f"score={score:+.3f}") + + # --- 6. Global directional bias --------------------------------------- + mean_r = statistics.mean(residuals) + median_r = statistics.median(residuals) + w_res = [residuals[i] for i in range(n) if stm[i] == 'w'] + b_res = [residuals[i] for i in range(n) if stm[i] == 'b'] + print(f"\n=== Global bias: typhoon(white) - crowd_median(white) ===\n") + print(f" overall: mean={mean_r:+.3f} median={median_r:+.3f} n={n}") + if w_res: + print(f" white to move: mean={statistics.mean(w_res):+.3f} " + f"median={statistics.median(w_res):+.3f} n={len(w_res)}") + if b_res: + print(f" black to move: mean={statistics.mean(b_res):+.3f} " + f"median={statistics.median(b_res):+.3f} n={len(b_res)}") + print(" (a nonzero overall mean/median with the same sign on " + "both stm splits points at a constant-ish term -- tempo, draw " + "score, a global scale factor -- rather than a positional one)") + + # --- 7. Residual by material phase ------------------------------------ + # Cheap, self-contained phase proxy from the FEN alone (not typhoon's + # own internal phase bucketing) so this works even without EVAL_DUMP + # internals: sum of N=3/B=3/R=5/Q=9 for both sides, pawns/kings + # excluded. Eval bugs are usually phase-local, and a flat top-K list + # hides that. + piece_val = {'n': 3, 'b': 3, 'r': 5, 'q': 9} + phases = [] + for fen in fens: + board = fen.split()[0] + phases.append(sum(piece_val.get(ch.lower(), 0) for ch in board if ch.isalpha())) + buckets = [(0, 12, "endgame (<=12)"), (13, 26, "middlegame (13-26)"), + (27, 999, "opening/near-full (>=27)")] + print(f"\n=== Residual by material phase (non-pawn point count, both sides) ===\n") + for lo, hi, name in buckets: + idxs = [i for i in range(n) if lo <= phases[i] <= hi] + if not idxs: + continue + r = [residuals[i] for i in idxs] + print(f" {name:28s} mean={statistics.mean(r):+.3f} " + f"median={statistics.median(r):+.3f} n={len(idxs)}") + + +if __name__ == "__main__": + main() diff --git a/src/eval_tune/match_play.py b/src/eval_tune/match_play.py index 5033cad..34a7917 100755 --- a/src/eval_tune/match_play.py +++ b/src/eval_tune/match_play.py @@ -96,7 +96,13 @@ _FIRST_MOVE_BETA_RE = re.compile( # count, (4) the rest of the line: the SAN PV text (WalkPV, util.c), or # "FL --" for a fail-low line (no PV -- the move that failed low isn't # necessarily the move eventually played) or "<SAN> ++" for a fail-high. -_ITER_RE = re.compile(r"^\s*(\d+)[+-]?\s+(-?\d+)\s+\S+\s+(\d+)\s*(.*)$") +# 2026-10-17: score group widened from (-?\d+) to ([+-]?\d+) -- typhoon +# itself never prints a leading "+" on a non-negative score (bare +# digits), but a UCI engine driven through PolyGlot's xboard adapter +# (e.g. Stockfish, see stockfish_xboard) DOES print an explicit "+" +# ("23 +28 198 700479 e4 c5"), which the old pattern rejected outright +# -- confirmed via direct regex test before this fix, not assumed. +_ITER_RE = re.compile(r"^\s*(\d+)[+-]?\s+([+-]?\d+)\s+\S+\s+(\d+)\s*(.*)$") # Reading PostMoveSearchReport's trailing block used to stop either at a # bounded line count or at gamelist.c's "_white(move N)_"/"_black(move N)_" # prompt -- both unreliable: the block's line count depends on whether the @@ -307,6 +313,13 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, white_stats = EngineStats() black_stats = EngineStats() white_last_fmb = black_last_fmb = None + # Tracked across the whole engine-interaction block below so the + # BrokenPipeError handler (see try/except wrapping it) can report + # which side/ply was in flight even when the pipe dies on a write + # that isn't inside the per-ply move loop (e.g. one of the startup + # sends, or a time-control send before the first ply). + ply_idx = -1 + white_to_move = None # Default Engine() spawns with NO --hash/--cpus at all, which means # a tiny 65536-entry hash table (main.c's memset-zero default) -- # far smaller than the 256m used everywhere else in this project's @@ -342,7 +355,8 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, # and the default behavior of a harness in active use should not # change silently underneath a running experiment. startup = ["book name /none", - "set ThinkOnOpponentsTime false"] + "set ThinkOnOpponentsTime false", + "set PostThrottle 0.0"] if post: startup.append("post") for command in startup: @@ -416,6 +430,7 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, tail = im.group(4).strip() if (last_iter_depth is not None and last_iter_nodes is not None and + last_iter_nodes > 0 and iter_nodes > last_iter_nodes and iter_depth > last_iter_depth): iter_log_ratio_sum += (math.log(iter_nodes / last_iter_nodes) @@ -434,10 +449,12 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, last_eval_score = iter_score last_eval_pv = tail[:-3].rstrip() if tail.endswith(" ++") else tail + abort_reason = None for _ in range(400): line = mover.stdout.readline() if not line: _dbg(tag, "EOF while scanning for move line") + abort_reason = "eof" break _dbg(tag, f"read(move-scan): {line!r}") _note_iter(line) @@ -445,15 +462,67 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, # Engine resigned instead of moving -- root.c prints # this and returns without ever printing "move ...", # so stop reading now rather than blocking forever. + # This IS a legitimate decisive result (unlike the + # other abort_reason cases below) -- resignation is a + # real chess outcome, not a harness/engine failure. + abort_reason = "resign" break m = _MOVE_RE.match(line) if m: - move_uci = m.group(1) + # root.c's "move ..." line is xboard/ICS-style + # (MoveToIcs, ics.c), which prints the promotion + # piece UPPERCASE ("e7e8Q") -- correct for that + # protocol, but chess.Move.from_uci() below enforces + # strict lowercase UCI and raises ValueError on it. + # Unlowered, EVERY promotion move got misclassified + # as illegal and the game aborted right as the + # promoting side (almost always the winning side in + # a long endgame) was about to seal the win -- this + # was the actual root cause of the ~47%-corrupted + # self-play corpus (docs/NNUE.md, 2026-10-17), not an + # engine stall/crash. Squares/files are already + # lowercase, so a blanket .lower() is safe here. + move_uci = m.group(1).lower() break + else: + # Read 400 lines without ever seeing "move ..." (and + # without EOF or resignation either) -- most likely an + # extremely verbose iteration output (deep search in a + # sparse endgame can print far more than 400 lines for a + # single move) exceeding this loop's hard cap while the + # engine is still alive and about to answer. + abort_reason = "line-cap-exceeded" if move_uci is None: - # side to move had nothing to play -- treat as loss for mover - result = 0.0 if white_to_move else 1.0 - return result, moves, move_evals, white_stats, black_stats + if abort_reason == "resign": + # side to move resigned -- legitimate loss for mover + result = 0.0 if white_to_move else 1.0 + return (result, moves, move_evals, white_stats, + black_stats, None) + # 2026-10-17: a real bug was found producing exactly this + # path for a non-resign reason (eof / line-cap) on a + # position where the mover was completely winning -- + # resignation was disabled for this corpus, so there is + # no legitimate explanation for move_uci staying None + # other than resign. The OLD code silently recorded this + # as an ordinary loss for the mover, which corrupted + # ~47% of a self-play training corpus with inverted + # results (see docs/NNUE.md). Never do that again: signal + # the caller with an aborted (result=None) game instead + # of guessing a winner, and capture enough context here + # to investigate later without needing to reproduce it + # live. + exit_code = mover.proc.poll() + abort_info = { + "reason": abort_reason, + "mover": "white" if white_to_move else "black", + "ply": ply_idx, + "fen_before_ply": board.fen(), + "mover_exit_code": exit_code, + "mover_alive": exit_code is None, + } + _dbg(tag, f"ABORT (not scored): {abort_info}") + return (None, moves, move_evals, white_stats, black_stats, + abort_info) # PostMoveSearchReport's stats block trails the "move" line # (confirmed against root.c and real output), not the other @@ -512,14 +581,32 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, else: black_last_fmb = ply_fmb + move = None try: move = chess.Move.from_uci(move_uci) - if move not in board.legal_moves: - result = 0.0 if white_to_move else 1.0 - return result, moves, move_evals, white_stats, black_stats + illegal = move not in board.legal_moves except ValueError: - result = 0.0 if white_to_move else 1.0 - return result, moves, move_evals, white_stats, black_stats + illegal = True + if illegal: + # The engine printed a "move ..." line, but it wasn't a + # legal move -- same reasoning as the move_uci-is-None + # case above: a real engine bug (producing garbage) + # deserves the same abort-not-score treatment, not a + # confidently-recorded loss with no diagnostic trail. + exit_code = mover.proc.poll() + abort_info = { + "reason": "illegal-move", + "mover": "white" if white_to_move else "black", + "ply": ply_idx, + "fen_before_ply": board.fen(), + "illegal_move_uci": move_uci, + "mover_exit_code": exit_code, + "mover_alive": exit_code is None, + } + _dbg(tag, f"ABORT (not scored): {abort_info}") + return (None, moves, move_evals, white_stats, black_stats, + abort_info) + assert move is not None # illegal is False only when the try succeeded board.push(move) moves.append(move_uci) score_white = None @@ -536,7 +623,7 @@ def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, result = 0.5 else: result = 1.0 if outcome.winner == chess.WHITE else 0.0 - return result, moves, move_evals, white_stats, black_stats + return result, moves, move_evals, white_stats, black_stats, None def _format_eval_comment(score_white, pv, depth): @@ -576,7 +663,15 @@ def build_pgn_game(start_fen, moves, result_white, white_name, black_name, game.headers["FEN"] = start_fen game.headers["Date"] = f"{date.today()}" game.setup(board) - result_str = {1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white] + # result_white is None for an ABORTED game (play_one_game's mover + # failed to produce a legal move for a non-resign reason -- see its + # abort_info return value) -- write a standard "unknown result" PGN + # tag rather than guessing a winner. fit_k_and_label.py's + # result_to_white_score() already treats "*" as "unknown, skip this + # game" for training purposes, so this alone keeps an aborted game + # out of any future training corpus without touching that script. + result_str = ({1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white] + if result_white is not None else "*") game.headers["Result"] = result_str node = game for i, move_uci in enumerate(moves): @@ -888,6 +983,48 @@ def main(): "Without this, local_ebf reads 0.000 -- the " "engines never print those lines because " "Engine.__init__ sends 'nopost'.") + ap.add_argument("--candidate-name", default="candidate", + help="PGN White/Black name recorded for the " + "candidate engine (default: 'candidate'). " + "2026-10-16: set this to something identifying " + "the eval/config actually used (e.g. " + "'candidate_nnue_loose') whenever the engine " + "isn't a plain HCE build -- self_play_games.pgn " + "is reused as NNUE training data later, and a " + "generic 'candidate' name makes it impossible " + "to tell which games came from a differently-" + "calibrated evaluator (whose scores shouldn't " + "be trusted as training labels) without " + "externally remembering run dates.") + ap.add_argument("--baseline-name", default="baseline", + help="PGN White/Black name recorded for the " + "baseline/head engine (default: 'baseline'); " + "same reasoning as --candidate-name.") + ap.add_argument("--no-color-swap", action="store_true", + help="play each sampled opening ONCE instead of " + "twice with colors swapped between head/" + "candidate. The swap exists to cancel out " + "color-dependent FIRST-MOVE ADVANTAGE in an " + "Elo-gating match; it is pure waste -- a byte-" + "for-byte duplicate game, just relabeled -- " + "whenever head_engine and candidate_engine are " + "the SAME binary (as every self-play " + "training-data run via " + "hce_training_corpus.sh is), because search " + "here is deterministic (--cpus 1) given a " + "fixed FEN+side-to-move, so swapping which " + "name is 'white'/'black' changes no move " + "actually played. Confirmed empirically " + "2026-10-16: consecutive game pairs in " + "nnue_self_play_training_data.pgn were exact " + "duplicates (identical FEN, identical move " + "list). Use this flag for self-play corpus " + "generation -- sample --games distinct " + "openings and play each once, rather than " + "--games/2 openings played twice -- and keep " + "the swap (the default) for real head-vs-" + "candidate Elo gating, where two different " + "binaries need it to cancel color bias.") ap.add_argument("--max-plies", type=int, default=200) ap.add_argument("--scratch", default="/usr/local/tmp/typhoon_match") ap.add_argument("--log", default="match_history.log") @@ -968,13 +1105,16 @@ def main(): head_engine = str(Path(args.head_engine).resolve()) candidate_engine = str(Path(args.candidate_engine).resolve()) - n_pairs = args.games // 2 - openings = sample_openings(args.pgn, n_pairs, seed=args.opening_seed) - n_games = len(openings) * 2 + if args.no_color_swap: + n_openings = args.games + else: + n_openings = args.games // 2 + openings = sample_openings(args.pgn, n_openings, seed=args.opening_seed) jobs = [] for fen in openings: - for candidate_is_white in (True, False): + seats = (True,) if args.no_color_swap else (True, False) + for candidate_is_white in seats: i = len(jobs) white_engine = candidate_engine if candidate_is_white else head_engine black_engine = head_engine if candidate_is_white else candidate_engine @@ -982,6 +1122,7 @@ def main(): candidate_points = 0.0 wins = draws = losses = 0 + aborted = 0 sprt = Sprt(args.elo0, args.elo1, args.alpha, args.beta) if args.sprt else None sprt_decision = None candidate_stats = EngineStats() @@ -1002,22 +1143,56 @@ def main(): if args.sd is None and head_time != cand_time: assert ((white_time == cand_time) == candidate_is_white), \ "time-odds seat mapping inverted" - result_white, moves, move_evals, white_stats, black_stats = play_one_game( - white_engine, black_engine, f"{args.scratch}/g{i}", fen, - depth=args.sd, - white_time_sec=white_time, black_time_sec=black_time, - max_plies=args.max_plies, post=args.post, - ) + try: + result_white, moves, move_evals, white_stats, black_stats, abort_info = ( + play_one_game( + white_engine, black_engine, f"{args.scratch}/g{i}", fen, + depth=args.sd, + white_time_sec=white_time, black_time_sec=black_time, + max_plies=args.max_plies, post=args.post, + ) + ) + except (BrokenPipeError, OSError) as exc: + # One of the two engine subprocesses died mid-game (crashed, + # was OOM-killed, etc.) -- play_one_game's own abort_info + # machinery only covers READ-side failures (EOF/line-cap + # while scanning for a move); a WRITE to an already-dead + # process's stdin raises BrokenPipeError straight out of + # Engine._send instead, uncaught anywhere below this. Left + # unhandled, that exception propagates out of this worker + # thread's run() through the Future and kills the entire + # match (every other in-flight/queued game lost) over a + # single flaky engine process -- confirmed live against a + # polyglot/Stockfish side via stockfish_xboard. Treat it the + # same way as any other abort: no result guessed, game + # skipped, run continues. + result_white = None + moves, move_evals = [], [] + white_stats, black_stats = EngineStats(), EngineStats() + abort_info = { + "reason": "engine-pipe-broken", + "exception": f"{type(exc).__name__}: {exc}", + } game_elapsed = time.monotonic() - game_start - candidate_result = result_white if candidate_is_white else (1.0 - result_white) - white_name = "candidate" if candidate_is_white else "baseline" - black_name = "baseline" if candidate_is_white else "candidate" + # result_white is None for an aborted game (play_one_game's mover + # failed to produce a legal move for a non-resign reason) -- do + # NOT guess a winner (see abort_info's own comment at the + # return sites in play_one_game for why this matters: it's + # exactly the bug that silently corrupted ~47% of an earlier + # self-play corpus). Propagate None through candidate_result too + # so run()'s caller can skip it rather than score it. + candidate_result = ( + None if result_white is None else + result_white if candidate_is_white else (1.0 - result_white) + ) + white_name = args.candidate_name if candidate_is_white else args.baseline_name + black_name = args.baseline_name if candidate_is_white else args.candidate_name game, _ = build_pgn_game(fen, moves, result_white, white_name, black_name, move_evals=move_evals) this_candidate_stats = white_stats if candidate_is_white else black_stats this_head_stats = black_stats if candidate_is_white else white_stats return (i, candidate_result, game, game_elapsed, - this_candidate_stats, this_head_stats) + this_candidate_stats, this_head_stats, abort_info) game_durations = [] job_iter = iter(jobs) @@ -1057,14 +1232,39 @@ def main(): finished, in_flight = wait(in_flight, return_when=FIRST_COMPLETED) for fut in finished: (i, candidate_result, game, game_elapsed, - this_candidate_stats, this_head_stats) = fut.result() + this_candidate_stats, this_head_stats, abort_info) = fut.result() + print(game, file=pgn_out, end="\n\n") + pgn_out.flush() + + if candidate_result is None: + # Aborted game (play_one_game's mover failed to + # produce a legal move for a non-resign reason -- + # see abort_info). Do NOT score it (no wins/ + # losses/draws/candidate_points/SPRT update, no + # contribution to the `done` denominator used for + # score/ETA) -- this is exactly the "silently + # record a decisive result with no diagnostic + # trail" bug that corrupted ~47% of an earlier + # self-play corpus (docs/NNUE.md, 2026-10-17). + # The game is still written to pgn_out above + # (Result "*", which fit_k_and_label.py's + # result_to_white_score() already treats as + # "unknown, skip" for training) so it isn't lost + # for post-mortem debugging, just not scored. + aborted += 1 + log.write(f"game {i}: ABORTED (not scored) " + f"abort_info={abort_info} " + f"elapsed={game_elapsed:.1f}s\n") + log.flush() + print(f" game {i}: ABORTED -- {abort_info}", + file=sys.stderr) + continue + done += 1 candidate_points += candidate_result game_durations.append(game_elapsed) candidate_stats.merge(this_candidate_stats) head_stats.merge(this_head_stats) - print(game, file=pgn_out, end="\n\n") - pgn_out.flush() if candidate_result == 1.0: wins += 1 elif candidate_result == 0.0: @@ -1131,7 +1331,7 @@ def main(): "(not waiting for in-flight games to finish naturally):", file=sys.stderr) print_final_summary(done, candidate_points, wins, draws, - losses, candidate_stats, head_stats, + losses, aborted, candidate_stats, head_stats, sprt, sprt_decision, args) # os._exit, not sys.exit: skips the `with` block's normal # ThreadPoolExecutor teardown (wait=True by default -- would @@ -1143,17 +1343,21 @@ def main(): # the file objects' own __exit__. os._exit(130) - print_final_summary(done, candidate_points, wins, draws, losses, + print_final_summary(done, candidate_points, wins, draws, losses, aborted, candidate_stats, head_stats, sprt, sprt_decision, args) -def print_final_summary(done, candidate_points, wins, draws, losses, +def print_final_summary(done, candidate_points, wins, draws, losses, aborted, candidate_stats, head_stats, sprt, sprt_decision, args): """Shared by the normal end-of-match path and the ^C handler below -- same CANDIDATE_SCORE/CANDIDATE_STATS/HEAD_STATS block either way, just - computed from whatever `done` games actually completed.""" + computed from whatever `done` games actually completed. `aborted` + (not included in `done` or in any win/draw/loss/score arithmetic) is + the count of games where a mover failed to produce a legal move for + a non-resign reason -- see play_one_game's abort_info. Surfaced here + so an elevated count is visible instead of silently discarded.""" n_games = done score = candidate_points / n_games if n_games else 0.5 elo, err = elo_diff(score, n_games) @@ -1182,10 +1386,11 @@ def print_final_summary(done, candidate_points, wins, draws, losses, f"cand={args.cand_time}s)") else: tc_str = f" TC=st{args.head_time}" + aborted_str = f" ABORTED={aborted}" if aborted else "" print(f"CANDIDATE_SCORE={score:.4f} GAMES={n_games} " f"WINS={wins} DRAWS={draws} LOSSES={losses} " f"ELO={elo:+.1f}+/-{err:.1f} LOWER95={lower95:.4f}" - f"{tc_str}{sprt_str}") + f"{tc_str}{sprt_str}{aborted_str}") print(f"CANDIDATE_STATS: {candidate_stats.summary()}") print(f"HEAD_STATS: {head_stats.summary()}") diff --git a/src/eval_tune/nnue_vs_head.sh b/src/eval_tune/nnue_vs_head.sh new file mode 100755 index 0000000..25a4c4f --- /dev/null +++ b/src/eval_tune/nnue_vs_head.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# +# Gated self-play match against a baseline binary. Historically this +# always compared the live working-tree build against head_reference/ +# -- generalized (2026-09-14) to take both binary paths as args so two +# arbitrary builds (e.g. a file-storm-only worktree build vs. the +# current working tree, to isolate one change at a time when several +# have landed together -- see king_safety_data/HANDOFF.md's session +# notes on not measuring two eval changes in one match) can be compared +# without hand-rolling a match_play.py invocation each time. +# +# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label] +# +# head_binary Baseline engine. Defaults to head_reference/typhoon +# (the usual "is this better than last checkpoint" +# question). +# candidate_binary Candidate engine. Defaults to a fresh copy of the +# live working-tree build (../typhoon) into +# /tmp/typhoon/typhoon_match_play_candidate -- pass +# an explicit path (already built) to skip that +# copy/freeze step entirely, e.g. when comparing +# two prebuilt worktree binaries. +# label Distinguishes scratch dir/log/pgn-out so multiple +# matches can run concurrently without colliding. +# Defaults to "match". +# +# ...and any further arguments are passed straight through to +# match_play.py, so a one-off flag no longer needs this file edited. +# +# Three env vars override the baked-in defaults (2026-10-03), so the +# common knobs are reachable without touching the invocation below: +# +# TC_ARGS time control. Default: "--st 4" +# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5" +# GAMES game cap. Default: 10000 +# +# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5" +# cannot resolve the effect sizes this project's LMR work actually +# produces. At the observed draw rate the standard error in Elo is about +# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000 +# games (~53h here) -- and every LMR match run to date has instead +# wandered around llr 0 for 2,000+ games and been abandoned, which is +# exactly what a true effect sitting BETWEEN the two hypotheses looks +# like. For a change you would only ship at >= 15 Elo, use +# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours +# instead of never. +# +# Examples: +# test_vs_head.sh # old default behavior +# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight +# +# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo: +# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh +# +# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the +# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in +# # THIS engine, which bounds what any depth-buying search change +# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity +# # rather than testing a hypothesis, and ELO=+/- prints either way. +# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo +# # of precision already separates "a ply is worth 25" from "worth 65". +# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \ +# test_vs_head.sh ../../head_reference/typhoon \ +# ../../head_reference/typhoon timeodds +# +# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/ +# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped +# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS" +# ahead of whatever args match_play.py's Engine class passes it -- +# that's the only change needed, since Engine spawns the given path +# directly via Popen (no shell), so a shebang script in its place works +# unmodified. HEAD is untouched, so it keeps running the classic HCE -- +# this is how you get an NNUE-candidate-vs-HCE-reference match without +# patching match_play.py's Engine (which currently passes identical +# engine_args to both sides): +# +# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh +# +# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by +# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/ +# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to +# also be set (the probe gate only does anything once NNUE eval itself +# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the +# same wrapper, enabling the lazy-eval accumulator probe gate at +# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE +# automatically, same convention as --nnueweights): +# +# NNUE_WEIGHTS=/path/to/net.nnue \ +# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \ +# test_vs_head.sh + +set -e + +# Only consume LEADING args that don't look like a flag as the three +# positionals (head/candidate/label) -- 2026-10-19 fix: a bare +# ./nnue_vs_head.sh --sd 10 used to land "--sd" in HEAD and "10" in +# CANDIDATE_ARG (since the old code unconditionally took "$1"/"$2"/"$3" +# regardless of content), the exact same footgun class the comment +# below already describes once biting the --shift-then-test-"$2" +# version of this script. A flag-looking first arg now means "no +# positionals given at all, everything is pass-through" instead. +POSITIONALS=() +while [ $# -gt 0 ] && [ "${#POSITIONALS[@]}" -lt 3 ]; do + case "$1" in + -*) break ;; + *) POSITIONALS+=("$1"); shift ;; + esac +done + +HEAD="${POSITIONALS[0]:-../../head_reference/typhoon}" +CANDIDATE_ARG="${POSITIONALS[1]:-}" +LABEL="${POSITIONALS[2]:-match}" +# Note the `-` (not `:-`) for the three env overrides below: an +# explicitly EMPTY value must be honored, not replaced by the default. +# SPRT_ARGS="" is the documented way to run without a hypothesis test +# (e.g. the time-odds calibration above), and `:-` would silently put +# --sprt back. +TC_ARGS="${TC_ARGS---st 4}" +SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}" +GAMES="${GAMES-10000}" + +# If the caller already passed --sd or --st through as an extra +# argument (e.g. ./nnue_vs_head.sh --sd 10), don't ALSO inject the +# baked-in TC_ARGS ("--st 4" by default) -- match_play.py's --sd/--st +# are mutually exclusive, so the two would collide +# ("error: argument --st: not allowed with argument --sd") even though +# the caller only asked for one of them. Pass-through wins; TC_ARGS is +# only the fallback when the caller didn't specify either. +for arg in "$@"; do + case "$arg" in + --sd|--st) TC_ARGS="" ;; + esac +done + +if [ -n "$CANDIDATE_ARG" ]; then + CANDIDATE="$CANDIDATE_ARG" +else + cd .. + gmake clean + gmake release MP=1 PERF_COUNTERS=1 + cd eval_tune + CANDIDATE=/tmp/typhoon/typhoon_match_play_candidate + cp -f ../typhoon_nnue ${CANDIDATE} + # typhoon_nnue's own TYPHOON_BIN/NNUE_WEIGHTS defaults are computed + # relative to ITS OWN location ("next to this script" -- see its + # header comment) -- correct when it's the real src/typhoon_nnue, + # but wrong once copied to /tmp/typhoon/ above: it would then look + # for the engine binary at /tmp/typhoon/typhoon and the weights + # file under /tmp/annotate_pgns/..., neither of which exist, and + # exit 1 immediately on every single spawn (2026-10-19: this is + # what produced hundreds of instant "BrokenPipeError" game aborts + # the first time this ran -- the copied wrapper was exiting before + # ever speaking the xboard protocol). match_play.py's Popen + # inherits this script's environment, so exporting the correct + # ABSOLUTE paths here -- computed from this script's own known-good + # location, not the relocated copy's -- fixes it without needing + # to touch typhoon_nnue itself. Only fill in NNUE_WEIGHTS if the + # caller didn't already set one explicitly. + export TYPHOON_BIN="$(cd .. && pwd)/typhoon" + : "${NNUE_WEIGHTS:=$(cd ../../annotate_pgns && pwd)/runs/sf98k/nnue_ckpt/nnue_weights.nnue}" + export NNUE_WEIGHTS +fi + +if [ -n "${NNUE_WEIGHTS:-}" ]; then + NNUE_WRAPPER=/tmp/typhoon/typhoon_match_play_candidate_nnue + NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\"" + if [ -n "${NNUE_PROBE_WEIGHTS:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\"" + fi + # NNUE_LAZY_MARGIN / NNUE_SUPERLAZY_MARGIN: 2026-10-16 looser- + # margin experiment (see docs/NNUE.md and command.c's "nnue + # lazymargin"/"nnue superlazyon"). NNUE_LAZY_MARGIN overrides every + # material bucket's lazy-probe margin with one flat cp value + # instead of the calibrated p98 table; NNUE_SUPERLAZY_MARGIN turns + # on the even-cheaper material-only pre-filter (off by default) + # with the given flat cp margin. Both require NNUE_WEIGHTS; the + # lazy one further requires NNUE_PROBE_WEIGHTS to do anything. + if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN" + fi + if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN" + fi + cat > "$NNUE_WRAPPER" <<WRAPPER +#!/usr/bin/env bash +exec "$CANDIDATE" $NNUE_FLAGS "\$@" +WRAPPER + chmod +x "$NNUE_WRAPPER" + CANDIDATE="$NNUE_WRAPPER" +fi + +# NNUE_CANDIDATE_NAME: PGN White/Black identity recorded for the +# candidate engine, forwarded to match_play.py's --candidate-name +# (2026-10-16, see that flag's own comment). Defaults to a name that +# flags "don't trust this as training data" whenever NNUE is in play, +# since self_play_games.pgn is reused later to train the NNUE net +# itself and a differently-calibrated evaluator's scores shouldn't be +# mixed in as labels without this being mechanically filterable. +# Override explicitly if a given NNUE config's scores are actually +# meant to be trusted. +NAME_ARGS="" +if [ -n "${NNUE_WEIGHTS:-}" ]; then + NNUE_CANDIDATE_NAME="${NNUE_CANDIDATE_NAME:-candidate_nnue}" + NAME_ARGS="--candidate-name $NNUE_CANDIDATE_NAME" +fi + +python3 ./match_play.py \ + ${HEAD} \ + ${CANDIDATE} \ + --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \ + --games ${GAMES} \ + --workers 18 \ + --post \ + ${TC_ARGS} \ + ${SPRT_ARGS} \ + ${NAME_ARGS} \ + --scratch /usr/local/tmp/typhoon_match_${LABEL} \ + --log ${LABEL}.log \ + "$@" diff --git a/src/eval_tune/relabel_selfplay_pgn.py b/src/eval_tune/relabel_selfplay_pgn.py new file mode 100644 index 0000000..be14e22 --- /dev/null +++ b/src/eval_tune/relabel_selfplay_pgn.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +One-off (but reusable) maintenance tool: retroactively rename the +White/Black PGN header value for games matching a given Event + Date +list, in place, without touching anything else in the file (moves, +comments, other headers byte-for-byte unchanged). + +Why this exists, 2026-10-16: self_play_games.pgn is reused as NNUE +training data (see docs/NNUE.md). match_play.py recorded every +candidate engine as literally "candidate" regardless of what that +candidate actually was, so the 2026-10-15/2026-10-16 batches (an NNUE +candidate vs the HCE baseline, scores on the NNUE side compressed/ +uncalibrated -- see the engine-vs-engine eval discussion that same +day) are indistinguishable from any other "candidate" batch by header +alone. match_play.py now takes --candidate-name/--baseline-name to +avoid this going forward; this script fixes the games written before +that existed. + +Deliberately stream-edits the file line by line rather than round- +tripping through python-chess's PGN writer -- re-serializing ~190k +games risks reformatting move/comment text (whitespace, line wrapping) +that downstream positional parsers (extract_positions.py) may depend +on being byte-identical to how it was originally written. This script +only ever rewrites a `[White "..."]`/`[Black "..."]` header line, and +only when that game's own `[Date ...]`/`[Event ...]` headers already +matched the target. + +Usage: + python3 relabel_selfplay_pgn.py <pgn-file> \\ + --dates 2026-10-15 2026-10-16 \\ + --event "typhoon self-play match" \\ + --old-name candidate --new-name candidate_nnue_loose \\ + [--dry-run] + +Writes to a sibling temp file and atomically replaces the original +only after a full pass succeeds -- a failure partway through never +leaves the original file half-rewritten. +""" +import argparse +import os +import re +import sys + + +HEADER_RE = re.compile(r'^\[(\w+)\s+"((?:[^"\\]|\\.)*)"\]\s*$') + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("pgn_file") + ap.add_argument("--dates", nargs="+", required=True, + help="Date header values (as PGN prints them, e.g. " + "2026-10-15) whose games are in scope") + ap.add_argument("--event", required=True, + help="Event header value a game must also match") + ap.add_argument("--old-name", required=True, + help="White/Black header value to rename") + ap.add_argument("--new-name", required=True, + help="replacement header value") + ap.add_argument("--dry-run", action="store_true", + help="report what would change, write nothing") + args = ap.parse_args() + + target_dates = set(args.dates) + tmp_path = args.pgn_file + ".relabel_tmp" + + cur_date = None + cur_event = None + in_scope = False # true once we know this game's headers match + games_seen = 0 + games_touched = 0 + renames = 0 + + with open(args.pgn_file, "r", encoding="utf-8", errors="surrogateescape") as fin, \ + open(tmp_path, "w", encoding="utf-8", errors="surrogateescape") as fout: + for line in fin: + m = HEADER_RE.match(line) + if m: + tag, value = m.group(1), m.group(2) + if tag == "Event": + # New header block -> new game starting. + games_seen += 1 + cur_event = value + cur_date = None + in_scope = False + elif tag == "Date": + cur_date = value + in_scope = (cur_event == args.event and cur_date in target_dates) + elif tag in ("White", "Black") and in_scope and value == args.old_name: + renames += 1 + games_touched += 1 if tag == "White" else 0 # count once/game is enough + if not args.dry_run: + line = f'[{tag} "{args.new_name}"]\n' + fout.write(line) + + if args.dry_run: + os.remove(tmp_path) + print(f"[dry run] {games_seen} games scanned, {renames} header " + f"values would be renamed ({args.old_name!r} -> " + f"{args.new_name!r}) for Event={args.event!r} Date in " + f"{sorted(target_dates)}") + return + + os.replace(tmp_path, args.pgn_file) + print(f"{games_seen} games scanned, {renames} header values renamed " + f"({args.old_name!r} -> {args.new_name!r}) for Event={args.event!r} " + f"Date in {sorted(target_dates)}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/eval_tune/stockfish_training_corpus.sh b/src/eval_tune/stockfish_training_corpus.sh new file mode 100755 index 0000000..75ce029 --- /dev/null +++ b/src/eval_tune/stockfish_training_corpus.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +python3 ./match_play.py \ + ../stockfish_xboard ../stockfish_xboard \ + --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \ + --pgn-out ../../pgn/stockfish_self_play_training_data.pgn \ + --games 100000 \ + --workers 100 \ + --sd 10 \ + --post \ + --no-color-swap \ + --scratch /usr/local/tmp/typhoon_match_sf_corpus \ + --log /tmp/typhoon/sf_training_self_play.log diff --git a/src/eval_tune/test_vs_head.sh b/src/eval_tune/test_vs_head.sh index 71b2282..fa44d44 100755 --- a/src/eval_tune/test_vs_head.sh +++ b/src/eval_tune/test_vs_head.sh @@ -75,12 +75,41 @@ # engine_args to both sides): # # NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh +# +# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by +# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/ +# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to +# also be set (the probe gate only does anything once NNUE eval itself +# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the +# same wrapper, enabling the lazy-eval accumulator probe gate at +# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE +# automatically, same convention as --nnueweights): +# +# NNUE_WEIGHTS=/path/to/net.nnue \ +# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \ +# test_vs_head.sh set -e -HEAD="${1:-../../head_reference/typhoon}" -CANDIDATE_ARG="${2:-}" -LABEL="${3:-match}" +# Only consume LEADING args that don't look like a flag as the three +# positionals (head/candidate/label) -- 2026-10-19 fix: a bare +# ./test_vs_head.sh --sd 10 used to land "--sd" in HEAD and "10" in +# CANDIDATE_ARG (since the old code unconditionally took "$1"/"$2"/"$3" +# regardless of content), the exact same footgun class the comment +# below already describes once biting the --shift-then-test-"$2" +# version of this script. A flag-looking first arg now means "no +# positionals given at all, everything is pass-through" instead. +POSITIONALS=() +while [ $# -gt 0 ] && [ "${#POSITIONALS[@]}" -lt 3 ]; do + case "$1" in + -*) break ;; + *) POSITIONALS+=("$1"); shift ;; + esac +done + +HEAD="${POSITIONALS[0]:-../../head_reference/typhoon}" +CANDIDATE_ARG="${POSITIONALS[1]:-}" +LABEL="${POSITIONALS[2]:-match}" # Note the `-` (not `:-`) for the three env overrides below: an # explicitly EMPTY value must be honored, not replaced by the default. # SPRT_ARGS="" is the documented way to run without a hypothesis test @@ -90,18 +119,17 @@ TC_ARGS="${TC_ARGS---st 4}" SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}" GAMES="${GAMES-10000}" -# Drop the (up to) three positional args so "$@" holds only extra -# pass-through flags for match_play.py. This MUST come after the three -# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above: -# an earlier version of this shifted first and then tested "$2" to -# decide whether to rebuild, which (a) made every no-extra-args run -# take the rebuild branch and silently clobber ../typhoon with a -# different make profile, and (b) made the first pass-through FLAG VALUE -# land in "$2" and get used as the candidate BINARY PATH -- producing a -# baffling "PermissionError: ... Permission denied: <scratch dir>" from -# Popen trying to execute a directory. -for _ in 1 2 3; do - if [ $# -gt 0 ]; then shift; fi +# If the caller already passed --sd or --st through as an extra +# argument (e.g. ./test_vs_head.sh --sd 10), don't ALSO inject the +# baked-in TC_ARGS ("--st 4" by default) -- match_play.py's --sd/--st +# are mutually exclusive, so the two would collide +# ("error: argument --st: not allowed with argument --sd") even though +# the caller only asked for one of them. Pass-through wins; TC_ARGS is +# only the fallback when the caller didn't specify either. +for arg in "$@"; do + case "$arg" in + --sd|--st) TC_ARGS="" ;; + esac done if [ -n "$CANDIDATE_ARG" ]; then @@ -117,14 +145,47 @@ fi if [ -n "${NNUE_WEIGHTS:-}" ]; then NNUE_WRAPPER=/tmp/typhoon/typhoon_match_play_candidate_nnue + NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\"" + if [ -n "${NNUE_PROBE_WEIGHTS:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\"" + fi + # NNUE_LAZY_MARGIN / NNUE_SUPERLAZY_MARGIN: 2026-10-16 looser- + # margin experiment (see docs/NNUE.md and command.c's "nnue + # lazymargin"/"nnue superlazyon"). NNUE_LAZY_MARGIN overrides every + # material bucket's lazy-probe margin with one flat cp value + # instead of the calibrated p98 table; NNUE_SUPERLAZY_MARGIN turns + # on the even-cheaper material-only pre-filter (off by default) + # with the given flat cp margin. Both require NNUE_WEIGHTS; the + # lazy one further requires NNUE_PROBE_WEIGHTS to do anything. + if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN" + fi + if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN" + fi cat > "$NNUE_WRAPPER" <<WRAPPER #!/usr/bin/env bash -exec "$CANDIDATE" --nnueweights "$NNUE_WEIGHTS" "\$@" +exec "$CANDIDATE" $NNUE_FLAGS "\$@" WRAPPER chmod +x "$NNUE_WRAPPER" CANDIDATE="$NNUE_WRAPPER" fi +# NNUE_CANDIDATE_NAME: PGN White/Black identity recorded for the +# candidate engine, forwarded to match_play.py's --candidate-name +# (2026-10-16, see that flag's own comment). Defaults to a name that +# flags "don't trust this as training data" whenever NNUE is in play, +# since self_play_games.pgn is reused later to train the NNUE net +# itself and a differently-calibrated evaluator's scores shouldn't be +# mixed in as labels without this being mechanically filterable. +# Override explicitly if a given NNUE config's scores are actually +# meant to be trusted. +NAME_ARGS="" +if [ -n "${NNUE_WEIGHTS:-}" ]; then + NNUE_CANDIDATE_NAME="${NNUE_CANDIDATE_NAME:-candidate_nnue}" + NAME_ARGS="--candidate-name $NNUE_CANDIDATE_NAME" +fi + python3 ./match_play.py \ ${HEAD} \ ${CANDIDATE} \ @@ -134,6 +195,7 @@ python3 ./match_play.py \ --post \ ${TC_ARGS} \ ${SPRT_ARGS} \ + ${NAME_ARGS} \ --scratch /usr/local/tmp/typhoon_match_${LABEL} \ --log ${LABEL}.log \ "$@" diff --git a/src/eval_tune/typhoon_hce_training_corpus.sh b/src/eval_tune/typhoon_hce_training_corpus.sh new file mode 100755 index 0000000..1fc16d8 --- /dev/null +++ b/src/eval_tune/typhoon_hce_training_corpus.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# +# Gated self-play match against a baseline binary. Historically this +# always compared the live working-tree build against head_reference/ +# -- generalized (2026-09-14) to take both binary paths as args so two +# arbitrary builds (e.g. a file-storm-only worktree build vs. the +# current working tree, to isolate one change at a time when several +# have landed together -- see king_safety_data/HANDOFF.md's session +# notes on not measuring two eval changes in one match) can be compared +# without hand-rolling a match_play.py invocation each time. +# +# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label] +# +# head_binary Baseline engine. Defaults to a fresh copy of the +# SAME just-built working-tree binary the candidate +# uses (/tmp/typhoon/typhoon_match_play_head) -- +# this is self-play training-data generation, not a +# "better than last checkpoint" gate, so baseline +# and candidate must share every runtime fix/option +# (e.g. the PostThrottle >= fix); a stale +# head_reference/typhoon silently lacks those and +# produces training data with gaps that look like +# search bugs but are actually just an old binary. +# Pass an explicit path (already built) -- together +# with candidate_binary -- to skip the rebuild/copy +# step entirely, e.g. when comparing two prebuilt +# worktree binaries. +# candidate_binary Candidate engine. Defaults to a fresh copy of the +# live working-tree build (../typhoon) into +# /tmp/typhoon/typhoon_match_play_candidate. Must be +# given together with head_binary -- either both +# positionals are prebuilt paths, or neither is and +# both get rebuilt copies. +# label Distinguishes scratch dir/log/pgn-out so multiple +# matches can run concurrently without colliding. +# Defaults to "match". +# +# ...and any further arguments are passed straight through to +# match_play.py, so a one-off flag no longer needs this file edited. +# +# Three env vars override the baked-in defaults (2026-10-03), so the +# common knobs are reachable without touching the invocation below: +# +# TC_ARGS time control. Default: "--st 4" +# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5" +# GAMES game cap. Default: 10000 +# +# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5" +# cannot resolve the effect sizes this project's LMR work actually +# produces. At the observed draw rate the standard error in Elo is about +# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000 +# games (~53h here) -- and every LMR match run to date has instead +# wandered around llr 0 for 2,000+ games and been abandoned, which is +# exactly what a true effect sitting BETWEEN the two hypotheses looks +# like. For a change you would only ship at >= 15 Elo, use +# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours +# instead of never. +# +# Examples: +# test_vs_head.sh # old default behavior +# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight +# +# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo: +# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh +# +# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the +# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in +# # THIS engine, which bounds what any depth-buying search change +# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity +# # rather than testing a hypothesis, and ELO=+/- prints either way. +# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo +# # of precision already separates "a ply is worth 25" from "worth 65". +# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \ +# test_vs_head.sh ../../head_reference/typhoon \ +# ../../head_reference/typhoon timeodds +# +# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/ +# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped +# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS" +# ahead of whatever args match_play.py's Engine class passes it -- +# that's the only change needed, since Engine spawns the given path +# directly via Popen (no shell), so a shebang script in its place works +# unmodified. HEAD is untouched, so it keeps running the classic HCE -- +# this is how you get an NNUE-candidate-vs-HCE-reference match without +# patching match_play.py's Engine (which currently passes identical +# engine_args to both sides): +# +# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh +# +# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by +# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/ +# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to +# also be set (the probe gate only does anything once NNUE eval itself +# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the +# same wrapper, enabling the lazy-eval accumulator probe gate at +# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE +# automatically, same convention as --nnueweights): +# +# NNUE_WEIGHTS=/path/to/net.nnue \ +# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \ +# test_vs_head.sh + +set -e + +HEAD_ARG="${1:-}" +CANDIDATE_ARG="${2:-}" +LABEL="${3:-match}" +# Note the `-` (not `:-`) for the three env overrides below: an +# explicitly EMPTY value must be honored, not replaced by the default. +# SPRT_ARGS="" is the documented way to run without a hypothesis test +# (e.g. the time-odds calibration above), and `:-` would silently put +# --sprt back. +#TC_ARGS="${TC_ARGS---st 4}" +#SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}" +GAMES="${GAMES-10000}" + +# Drop the (up to) three positional args so "$@" holds only extra +# pass-through flags for match_play.py. This MUST come after the three +# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above: +# an earlier version of this shifted first and then tested "$2" to +# decide whether to rebuild, which (a) made every no-extra-args run +# take the rebuild branch and silently clobber ../typhoon with a +# different make profile, and (b) made the first pass-through FLAG VALUE +# land in "$2" and get used as the candidate BINARY PATH -- producing a +# baffling "PermissionError: ... Permission denied: <scratch dir>" from +# Popen trying to execute a directory. +for _ in 1 2 3; do + if [ $# -gt 0 ]; then shift; fi +done + +if [ -n "$HEAD_ARG" ] && [ -n "$CANDIDATE_ARG" ]; then + HEAD="$HEAD_ARG" + CANDIDATE="$CANDIDATE_ARG" +else + cd .. + gmake clean + gmake release MP=1 PERF_COUNTERS=1 + cd eval_tune + # Both sides get a fresh copy of the SAME just-built ../typhoon -- + # unlike head_reference/typhoon (a stale checked-in binary), this + # guarantees baseline and candidate share every runtime fix/option + # (e.g. the PostThrottle >= fix) so a missing comment in training + # data can't be blamed on baseline running old code. Pass both + # HEAD_ARG and CANDIDATE_ARG explicitly to compare two prebuilt + # binaries instead. + HEAD="${HEAD_ARG:-/tmp/typhoon/typhoon_match_play_head}" + CANDIDATE="${CANDIDATE_ARG:-/tmp/typhoon/typhoon_match_play_candidate}" + cp -f ../typhoon ${HEAD} + cp -f ../typhoon ${CANDIDATE} +fi + +# --no-color-swap: head and candidate are copies of the SAME binary +# here (this is corpus generation, not an Elo gate), so match_play.py's +# default color-swap-per-opening would just play every opening TWICE +# and produce a byte-for-byte duplicate game the second time (confirmed +# 2026-10-16 against nnue_self_play_training_data.pgn) -- pure wasted +# CPU for a training corpus that wants variety, not duplication. See +# --no-color-swap's own help text in match_play.py for the full +# reasoning. +python3 ./match_play.py \ + ${HEAD} \ + ${CANDIDATE} \ + --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \ + --pgn-out ../../pgn/nnue_self_play_training_data.pgn \ + --games ${GAMES} \ + --workers 26 \ + --sd 10 \ + --post \ + --no-color-swap \ + ${NAME_ARGS} \ + --scratch /usr/local/tmp/typhoon_match_${LABEL} \ + --log /tmp/typhoon/nnue_training_self_play.log \ + "$@" diff --git a/src/eval_tune/typhoon_nnue_training_corpus.sh b/src/eval_tune/typhoon_nnue_training_corpus.sh new file mode 100755 index 0000000..b1a00d4 --- /dev/null +++ b/src/eval_tune/typhoon_nnue_training_corpus.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# +# Gated self-play match against a baseline binary. Historically this +# always compared the live working-tree build against head_reference/ +# -- generalized (2026-09-14) to take both binary paths as args so two +# arbitrary builds (e.g. a file-storm-only worktree build vs. the +# current working tree, to isolate one change at a time when several +# have landed together -- see king_safety_data/HANDOFF.md's session +# notes on not measuring two eval changes in one match) can be compared +# without hand-rolling a match_play.py invocation each time. +# +# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label] +# +# head_binary Baseline engine. Defaults to a fresh copy of the +# SAME just-built working-tree binary the candidate +# uses (/tmp/typhoon/typhoon_match_play_head) -- +# this is self-play training-data generation, not a +# "better than last checkpoint" gate, so baseline +# and candidate must share every runtime fix/option +# (e.g. the PostThrottle >= fix); a stale +# head_reference/typhoon silently lacks those and +# produces training data with gaps that look like +# search bugs but are actually just an old binary. +# Pass an explicit path (already built) -- together +# with candidate_binary -- to skip the rebuild/copy +# step entirely, e.g. when comparing two prebuilt +# worktree binaries. +# candidate_binary Candidate engine. Defaults to a fresh copy of the +# live working-tree build (../typhoon) into +# /tmp/typhoon/typhoon_match_play_candidate. Must be +# given together with head_binary -- either both +# positionals are prebuilt paths, or neither is and +# both get rebuilt copies. +# label Distinguishes scratch dir/log/pgn-out so multiple +# matches can run concurrently without colliding. +# Defaults to "match". +# +# ...and any further arguments are passed straight through to +# match_play.py, so a one-off flag no longer needs this file edited. +# +# Three env vars override the baked-in defaults (2026-10-03), so the +# common knobs are reachable without touching the invocation below: +# +# TC_ARGS time control. Default: "--st 4" +# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5" +# GAMES game cap. Default: 10000 +# +# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5" +# cannot resolve the effect sizes this project's LMR work actually +# produces. At the observed draw rate the standard error in Elo is about +# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000 +# games (~53h here) -- and every LMR match run to date has instead +# wandered around llr 0 for 2,000+ games and been abandoned, which is +# exactly what a true effect sitting BETWEEN the two hypotheses looks +# like. For a change you would only ship at >= 15 Elo, use +# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours +# instead of never. +# +# Examples: +# test_vs_head.sh # old default behavior +# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight +# +# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo: +# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh +# +# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the +# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in +# # THIS engine, which bounds what any depth-buying search change +# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity +# # rather than testing a hypothesis, and ELO=+/- prints either way. +# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo +# # of precision already separates "a ply is worth 25" from "worth 65". +# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \ +# test_vs_head.sh ../../head_reference/typhoon \ +# ../../head_reference/typhoon timeodds +# +# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/ +# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped +# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS" +# ahead of whatever args match_play.py's Engine class passes it -- +# that's the only change needed, since Engine spawns the given path +# directly via Popen (no shell), so a shebang script in its place works +# unmodified. HEAD is untouched, so it keeps running the classic HCE -- +# this is how you get an NNUE-candidate-vs-HCE-reference match without +# patching match_play.py's Engine (which currently passes identical +# engine_args to both sides): +# +# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh +# +# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by +# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/ +# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to +# also be set (the probe gate only does anything once NNUE eval itself +# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the +# same wrapper, enabling the lazy-eval accumulator probe gate at +# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE +# automatically, same convention as --nnueweights): +# +# NNUE_WEIGHTS=/path/to/net.nnue \ +# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \ +# test_vs_head.sh + +set -e + +HEAD_ARG="${1:-}" +CANDIDATE_ARG="${2:-}" +LABEL="${3:-match}" +# Note the `-` (not `:-`) for the three env overrides below: an +# explicitly EMPTY value must be honored, not replaced by the default. +# SPRT_ARGS="" is the documented way to run without a hypothesis test +# (e.g. the time-odds calibration above), and `:-` would silently put +# --sprt back. +#TC_ARGS="${TC_ARGS---st 4}" +#SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}" +GAMES="${GAMES-10000}" + +# Drop the (up to) three positional args so "$@" holds only extra +# pass-through flags for match_play.py. This MUST come after the three +# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above: +# an earlier version of this shifted first and then tested "$2" to +# decide whether to rebuild, which (a) made every no-extra-args run +# take the rebuild branch and silently clobber ../typhoon with a +# different make profile, and (b) made the first pass-through FLAG VALUE +# land in "$2" and get used as the candidate BINARY PATH -- producing a +# baffling "PermissionError: ... Permission denied: <scratch dir>" from +# Popen trying to execute a directory. +for _ in 1 2 3; do + if [ $# -gt 0 ]; then shift; fi +done + +if [ -n "$HEAD_ARG" ] && [ -n "$CANDIDATE_ARG" ]; then + HEAD="$HEAD_ARG" + CANDIDATE="$CANDIDATE_ARG" +else + cd .. + gmake clean + gmake release MP=1 PERF_COUNTERS=1 + cd eval_tune + # Both sides get a fresh copy of the SAME just-built ../typhoon -- + # unlike head_reference/typhoon (a stale checked-in binary), this + # guarantees baseline and candidate share every runtime fix/option + # (e.g. the PostThrottle >= fix) so a missing comment in training + # data can't be blamed on baseline running old code. Pass both + # HEAD_ARG and CANDIDATE_ARG explicitly to compare two prebuilt + # binaries instead. + HEAD="${HEAD_ARG:-/tmp/typhoon/typhoon_match_play_nnue1}" + CANDIDATE="${CANDIDATE_ARG:-/tmp/typhoon/typhoon_match_play_nnue2}" + cp -f ../typhoon ${HEAD} + cp -f ../typhoon ${CANDIDATE} +fi + +# --no-color-swap: head and candidate are copies of the SAME binary +# here (this is corpus generation, not an Elo gate), so match_play.py's +# default color-swap-per-opening would just play every opening TWICE +# and produce a byte-for-byte duplicate game the second time (confirmed +# 2026-10-16 against nnue_self_play_training_data.pgn) -- pure wasted +# CPU for a training corpus that wants variety, not duplication. See +# --no-color-swap's own help text in match_play.py for the full +# reasoning. +NNUE_WEIGHTS="${NNUE_WEIGHTS:-$HOME/typhoon/annotate_pgns/runs/sf98k/nnue_ckpt/nnue_weights.nnue}" +if [ ! -f "$NNUE_WEIGHTS" ]; then + echo "typhoon_nnue: NNUE_WEIGHTS file not found: $NNUE_WEIGHTS" >&2 + exit 1 +fi + +NNUE_PROBE_WEIGHTS="${NNUE_PROBE_WEIGHTS-}" + +# Both HEAD and CANDIDATE are the same just-built binary (this is NNUE +# self-play training-data generation, not an HCE-vs-NNUE Elo gate), so +# the NNUE flags get wrapped into exec shims for BOTH sides -- a bare +# string-concatenation of path+flags doesn't work here: match_play.py's +# head_engine/candidate_engine are resolved via Path(...).resolve() as a +# single filesystem path, so passing "$HEAD $ARGS" as one argument would +# just produce a nonexistent path containing literal spaces. See +# test_vs_head.sh's NNUE_WRAPPER for the same pattern, used there to wrap +# only the candidate (HCE head vs NNUE candidate); here both sides need +# the net since both sides generate training data as NNUE. +NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\"" +if [ -n "$NNUE_PROBE_WEIGHTS" ] && [ -f "$NNUE_PROBE_WEIGHTS" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\"" + if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN" + fi +fi +if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then + NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN" +fi + +HEAD_WRAPPER=/tmp/typhoon/typhoon_match_play_nnue1_wrapped +cat > "$HEAD_WRAPPER" <<WRAPPER +#!/usr/bin/env bash +exec "$HEAD" $NNUE_FLAGS "\$@" +WRAPPER +chmod +x "$HEAD_WRAPPER" +HEAD="$HEAD_WRAPPER" + +CANDIDATE_WRAPPER=/tmp/typhoon/typhoon_match_play_nnue2_wrapped +cat > "$CANDIDATE_WRAPPER" <<WRAPPER +#!/usr/bin/env bash +exec "$CANDIDATE" $NNUE_FLAGS "\$@" +WRAPPER +chmod +x "$CANDIDATE_WRAPPER" +CANDIDATE="$CANDIDATE_WRAPPER" + +# See test_vs_head.sh's NNUE_CANDIDATE_NAME comment: flags the PGN +# White/Black identity so NNUE-evaluated self-play games are +# mechanically filterable out of training data that isn't meant to +# mix differently-calibrated evaluator scores. +NAME_ARGS="--candidate-name ${NNUE_CANDIDATE_NAME:-candidate_nnue} --baseline-name ${NNUE_HEAD_NAME:-head_nnue}" + +python3 ./match_play.py \ + "${HEAD}" \ + "${CANDIDATE}" \ + --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \ + --pgn-out ../../pgn/nnue_self_play_training_data.pgn \ + --games ${GAMES} \ + --workers 26 \ + --sd 10 \ + --post \ + --no-color-swap \ + ${NAME_ARGS} \ + --scratch /usr/local/tmp/typhoon_match_${LABEL} \ + --log /tmp/typhoon/nnue_training_self_play.log \ + "$@" |