diff options
Diffstat (limited to 'src/eval_tune/match_play.py')
| -rwxr-xr-x | src/eval_tune/match_play.py | 660 |
1 files changed, 660 insertions, 0 deletions
diff --git a/src/eval_tune/match_play.py b/src/eval_tune/match_play.py new file mode 100755 index 0000000..6891964 --- /dev/null +++ b/src/eval_tune/match_play.py @@ -0,0 +1,660 @@ +#!/usr/bin/env python3 +""" +Head-to-head match: head binary vs. candidate binary, N games, alternating +colors, opening positions sampled from a filtered TWIC pool for variety. + +Two separate compiled typhoon binaries (built from different commits/ +working trees), not one binary with two loaded `evaldna` files -- the +DNA-diffing design this replaced was built for the Texel auto-tuning +pipeline, which this project has since moved away from in favor of +hand-tuned eval constants baked directly into eval.c. Comparing two +binaries means a real build is required for each side before running +this (there's no live DNA swap), but it's the right model now: what's +actually being compared is two different eval.c/search.c source trees, +not two parameter files loaded into an otherwise-identical process. + +The authoritative board is kept in python-chess; each engine is treated +as stateless per move (force; setboard <fen>; go) rather than fed +"usermove", mirroring the go/force turn-taking already verified to work +in tune_eval_dna.py's play_selfplay_game. + +Usage: + python3 match_play.py <head_engine_path> <candidate_engine_path> \\ + --pgn ../../pgn/twic_filtered.pgn --games 500 --workers 4 --st 1 + + Add --sprt (with --elo0/--elo1/--alpha/--beta) to stop as soon as a + Sequential Probability Ratio Test concludes instead of always + playing exactly --games games -- see the Sprt class docstring for + why a fixed game count is usually the wrong tool for "are these two + engines really different in strength". + +Prints: + CANDIDATE_SCORE=<0..1> GAMES=<n> WINS=<a> DRAWS=<b> LOSSES=<c> ELO=<+/-x.x> + LOWER95=<x> [SPRT=... LLR=... BOUNDS=... ELO0=... ELO1=...] + CANDIDATE_STATS: nps=... ebf=... first_move_beta=...% nodes=... plies=... + HEAD_STATS: nps=... ebf=... first_move_beta=...% nodes=... plies=... +for the orchestrator to parse. Speed/tree-shape stats (EngineStats) are +pulled from the same per-move PostMoveSearchReport block every "go" +already prints, not a separate benchmark -- see EngineStats' docstring. +""" +import argparse +import hashlib +import io +import math +import random +import re +import sys +import time +from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED +from pathlib import Path + +import chess +import chess.pgn + +sys.path.insert(0, str(Path(__file__).parent)) +from tune_eval_dna import Engine # noqa: E402 + +_MOVE_RE = re.compile(r"^move (\S+)") +# PostMoveSearchReport (root.c) prints these AFTER the "move" line, not +# before -- confirmed against root.c (Trace("move %s\n", ...) at line +# ~1367/1369, PostMoveSearchReport(ctx) called at ~1384, strictly after) +# and against real engine output. A per-move stats block trails every +# move, not just script-run suite summaries -- see EBF/first-move-beta +# discussion in session history. +_SEARCHED_RE = re.compile( + r"^Searched for\s+([\d.]+) seconds, saw (\d+) nodes " + r"\((\d+) qnodes\) \(\s*([\d.]+) nps\)\.") +_DEPTH_RE = re.compile(r"^tellothers d(\d+),") +_FIRST_MOVE_BETA_RE = re.compile( + r"^First move beta cutoff rate was\s+([\d.]+) percent\.") +_STATS_TRAILING_LINES = 20 # bound on how far past "move" to keep reading + + +def sample_openings(pgn_path, n, min_ply=8, max_ply=20, seed=0, + cache_dir="opening_cache"): + """Pick n random (roughly) opening positions out of a pgn pool by + seeking to random byte offsets and parsing just the one game found + there, rather than sequentially chess.pgn.read_game()-ing from the + start -- on a several-hundred-MB pool, replaying every game up to + the sample point (even while mostly skipping them) means real I/O + and SAN-parsing cost that scales with how far into the file the + first few accepted samples happen to land, which was observed + taking minutes for as few as 2 requested openings. Cached like + tune_eval_dna.py's position_cache, keyed on pgn identity + params, + since repeated cycle.sh runs against the same pool shouldn't pay + this more than once.""" + st = Path(pgn_path).stat() + key = hashlib.sha256( + f"{pgn_path}|{st.st_mtime}|{st.st_size}|{n}|{min_ply}|{max_ply}|{seed}" + .encode() + ).hexdigest()[:16] + cache_path = Path(cache_dir) / f"openings_{key}.txt" + if cache_path.exists(): + openings = cache_path.read_text().splitlines() + print(f" loaded {len(openings)} openings from cache {cache_path}", + file=sys.stderr) + return openings + + rng = random.Random(seed) + file_size = st.st_size + openings = [] + attempts = 0 + max_attempts = n * 50 + 200 # generous; each attempt is one seek+one game parse + with open(pgn_path, "rb") as fb: + while len(openings) < n and attempts < max_attempts: + attempts += 1 + offset = rng.randint(0, max(file_size - 1, 0)) + fb.seek(offset) + fb.readline() # discard partial line at the seek point + # scan forward to the start of the next game's header block + start = fb.tell() + line = fb.readline() + while line and not line.startswith(b"[Event "): + start = fb.tell() + line = fb.readline() + if not line: + continue # landed past the last game in the file; retry elsewhere + # read forward to the START of the game AFTER this one (or EOF) + fb.seek(start) + fb.readline() # consume this game's own "[Event " line + end = fb.tell() + line = fb.readline() + while line and not line.startswith(b"[Event "): + end = fb.tell() + line = fb.readline() + fb.seek(start) + raw = fb.read(end - start).decode("utf-8", errors="replace") + game = chess.pgn.read_game(io.StringIO(raw)) + if game is None: + continue + moves = list(game.mainline_moves()) + if len(moves) < min_ply: + continue + board = game.board() + ply = rng.randint(min_ply, min(max_ply, len(moves))) + for mv in moves[:ply]: + board.push(mv) + openings.append(board.fen()) + + if len(openings) < n: + print(f"warning: only found {len(openings)} openings " + f"(wanted {n}) in {pgn_path}", file=sys.stderr) + + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_text("\n".join(openings) + "\n" if openings else "") + return openings + + +def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen, + depth=None, time_control_sec=None, max_plies=200): + """Returns (result, moves, white_stats, black_stats) where result is + 1.0/0.5/0.0 from White's perspective, moves is the list of UCI moves + actually played (for PGN reconstruction), and white_stats/black_stats + are EngineStats instances covering that side's moves in this game.""" + moves = [] + white_stats = EngineStats() + black_stats = EngineStats() + white_last_fmb = black_last_fmb = None + # Default Engine() spawns with NO --hash/--cpus at all, which means + # a tiny 65536-entry hash table (main.c's memset-zero default) -- + # far smaller than the 256m used everywhere else in this project's + # own testing (test.sh). A small hash causes much worse move + # ordering/more re-search in complex positions -- pass explicit + # options here so gate-match search timing is representative + # rather than an artifact of an unset default. + # --logfile matters here, not just cosmetically: the default is a + # shared "typhoon.log" in cwd (main.c's memset-zero default), + # opened via BackupFile()-then-fopen(wb+) at startup and fflush()ed + # on every Log()/Bug() call. With N concurrent engine processes (2 + # per game x --workers games) all doing that against the SAME + # file, you get real rename races and lock contention -- observed + # as unexplained match slowdown. "-" disables file logging + # entirely (main.c: skips BackupFile/fopen altogether) -- fine + # here since match_play.py never reads engine logs (moves come off + # stdout, games get their own match_games.pgn). + engine_args = ["--hash", "256m", "--cpus", "1", "--logfile", "-"] + with Engine(white_engine_path, f"{scratch_dir}/w", engine_args) as we, \ + Engine(black_engine_path, f"{scratch_dir}/b", engine_args) as be: + # Disable book and pondering. Set time control. Resign lost positions. + tc_cmd = f"sd {depth}" if depth is not None else f"st {time_control_sec}" + for command in ( "book name /none", + "set ThinkOnOpponentsTime false", + "set ResignThreshold -1000", + tc_cmd): + we._send(command) + be._send(command) + + board = chess.Board(fen) + for _ in range(max_plies): + if board.is_game_over(claim_draw=True): + break + white_to_move = board.turn == chess.WHITE + mover = we if white_to_move else be + mover_stats = white_stats if white_to_move else black_stats + mover._send("force") + mover._send(f"setboard {board.fen()}") + mover._send("go") + move_uci = None + for _ in range(400): + line = mover.stdout.readline() + if not line: + break + if line.startswith("tellics resign"): + # Engine resigned instead of moving -- root.c prints + # this and returns without ever printing "move ...", + # so stop reading now rather than blocking forever. + break + m = _MOVE_RE.match(line) + if m: + move_uci = m.group(1) + break + if move_uci is None: + # side to move had nothing to play -- treat as loss for mover + result = 0.0 if white_to_move else 1.0 + return result, moves, white_stats, black_stats + + # PostMoveSearchReport's stats block trails the "move" line + # (confirmed against root.c and real output), not the other + # way around -- keep reading a bounded number of further + # lines to pick it up before moving on to the next ply. + ply_nodes = ply_time = ply_depth = ply_fmb = None + for _ in range(_STATS_TRAILING_LINES): + line = mover.stdout.readline() + if not line: + break + sm = _SEARCHED_RE.match(line) + if sm: + ply_time = float(sm.group(1)) + ply_nodes = int(sm.group(2)) + continue + dm = _DEPTH_RE.match(line) + if dm: + ply_depth = int(dm.group(1)) + continue + fm = _FIRST_MOVE_BETA_RE.match(line) + if fm: + ply_fmb = float(fm.group(1)) + break # last line of the block we care about + if ply_nodes is not None and ply_time is not None: + mover_stats.add_ply(ply_nodes, ply_time, ply_depth) + if ply_fmb is not None: + if white_to_move: + white_last_fmb = ply_fmb + else: + black_last_fmb = ply_fmb + + try: + move = chess.Move.from_uci(move_uci) + if move not in board.legal_moves: + result = 0.0 if white_to_move else 1.0 + return result, moves, white_stats, black_stats + except ValueError: + result = 0.0 if white_to_move else 1.0 + return result, moves, white_stats, black_stats + board.push(move) + moves.append(move_uci) + + white_stats.add_game_first_move_beta(white_last_fmb, white_stats.plies) + black_stats.add_game_first_move_beta(black_last_fmb, black_stats.plies) + + outcome = board.outcome(claim_draw=True) + if outcome is None or outcome.winner is None: + result = 0.5 + else: + result = 1.0 if outcome.winner == chess.WHITE else 0.0 + return result, moves, white_stats, black_stats + + +def build_pgn_game(start_fen, moves, result_white, white_name, black_name): + board = chess.Board(start_fen) + game = chess.pgn.Game() + game.headers["Event"] = "typhoon DNA gate match" + game.headers["White"] = white_name + game.headers["Black"] = black_name + game.headers["FEN"] = start_fen + game.setup(board) + result_str = {1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white] + game.headers["Result"] = result_str + node = game + for move_uci in moves: + node = node.add_variation(chess.Move.from_uci(move_uci)) + return game, result_str + + +def elo_diff(score, n): + """Rough Elo estimate + a 95%-ish error bar from a win rate, treating + games as independent Bernoulli-ish trials on the score. Good enough + as a signal, not a substitute for a real SPRT.""" + score = min(max(score, 1e-6), 1 - 1e-6) + elo = -400 * math.log10(1 / score - 1) + # standard error on the score, propagated through the logit + se_score = math.sqrt(score * (1 - score) / max(n, 1)) + se_elo = (400 / math.log(10)) * se_score / (score * (1 - score)) + return elo, 1.96 * se_elo + + +class EngineStats: + """Per-engine (not per-color -- candidate/baseline swap sides every + other game) search-quality accumulator: speed and tree shape, pulled + from the same PostMoveSearchReport block every move already prints + (see _SEARCHED_RE/_DEPTH_RE/_FIRST_MOVE_BETA_RE), not a separate + benchmark. EBF is computed the same way script.c's suite runs do + (nodes**(1/depth) per move, see head_reference notes on why this can + diverge from first-move-beta-cutoff-rate when pruning/aspiration- + window re-searches inflate node counts independent of true tree + shape), then averaged across all moves an engine played. First-move + beta cutoff rate is read as a running total inside a single engine + process's lifetime (root.c never resets those counters mid-process), + and since play_one_game spawns a fresh process per game, the value + from an engine's last move of a game is already that whole game's + rate -- summed here across games as a plies-weighted average so a + long game doesn't count the same as a short one. + """ + + def __init__(self): + self.total_nodes = 0 + self.total_time = 0.0 + self.plies = 0 + self.ebf_sum = 0.0 + self.ebf_count = 0 + self._fmb_weighted_sum = 0.0 + self._fmb_weight = 0 + + def add_ply(self, nodes, elapsed, depth): + self.total_nodes += nodes + self.total_time += elapsed + self.plies += 1 + if depth and depth > 0 and nodes > 0: + self.ebf_sum += nodes ** (1.0 / depth) + self.ebf_count += 1 + + def add_game_first_move_beta(self, pct, plies_in_game): + if pct is not None and plies_in_game > 0: + self._fmb_weighted_sum += pct * plies_in_game + self._fmb_weight += plies_in_game + + def merge(self, other): + self.total_nodes += other.total_nodes + self.total_time += other.total_time + self.plies += other.plies + self.ebf_sum += other.ebf_sum + self.ebf_count += other.ebf_count + self._fmb_weighted_sum += other._fmb_weighted_sum + self._fmb_weight += other._fmb_weight + + @property + def nps(self): + return self.total_nodes / self.total_time if self.total_time else 0.0 + + @property + def avg_ebf(self): + return self.ebf_sum / self.ebf_count if self.ebf_count else 0.0 + + @property + def avg_first_move_beta(self): + return (self._fmb_weighted_sum / self._fmb_weight + if self._fmb_weight else 0.0) + + def summary(self): + return (f"nps={self.nps:.0f} ebf={self.avg_ebf:.3f} " + f"first_move_beta={self.avg_first_move_beta:.1f}% " + f"nodes={self.total_nodes} plies={self.plies}") + + +def elo_to_score(elo): + """Expected per-game score (0..1) for a side rated `elo` points above + an opponent, via the standard logistic Elo model.""" + return 1.0 / (1.0 + 10.0 ** (-elo / 400.0)) + + +class Sprt: + """Sequential Probability Ratio Test for engine-vs-engine gating, same + formulation fishtest/cutechess-cli use for exactly this problem: two + Elo hypotheses (H0: true strength gap is elo0 or worse, H1: elo1 or + better), tested via the log-likelihood ratio of a normal + approximation to the per-game trinomial (W/D/L) score, with variance + re-estimated from the running W/D/L mix as games accumulate. + + Unlike a fixed-N test (this script's original LOWER95 gate), this + stops as soon as the evidence is conclusive in either direction -- + a handful of games if the true effect is large, potentially tens of + thousands if it's genuinely marginal -- rather than committing to + one game count up front regardless of the actual effect size. + + See CLAUDE.md / the "how many games" discussion this was built from: + for a 95%/80%-power fixed-N test, resolving a 5 Elo gap needs + ~35-40k games, 10 Elo needs ~9-10k, 20-30 Elo needs ~1-2.5k. SPRT + doesn't remove that fundamental cost, it just avoids overpaying for + it when the true effect is far from the boundary either way. + """ + + def __init__(self, elo0, elo1, alpha=0.05, beta=0.05): + self.elo0 = elo0 + self.elo1 = elo1 + self.s0 = elo_to_score(elo0) + self.s1 = elo_to_score(elo1) + self.la = math.log(beta / (1 - alpha)) + self.lb = math.log((1 - beta) / alpha) + self.wins = self.draws = self.losses = 0 + + def update(self, result): + """result: 1.0 (candidate win), 0.5 (draw), or 0.0 (candidate loss).""" + if result == 1.0: + self.wins += 1 + elif result == 0.0: + self.losses += 1 + else: + self.draws += 1 + + @property + def n(self): + return self.wins + self.draws + self.losses + + def llr(self): + n = self.n + if n == 0: + return 0.0 + mean = (self.wins + 0.5 * self.draws) / n + var = (self.wins * (1 - mean) ** 2 + + self.draws * (0.5 - mean) ** 2 + + self.losses * (0 - mean) ** 2) / n + if var <= 0: + # every game so far had the identical result -- LLR is only + # defined once there's some spread; treat as inconclusive. + return 0.0 + return n * (self.s1 - self.s0) * (mean - (self.s0 + self.s1) / 2) / var + + def decision(self): + """Returns 'H1' (accept candidate is >= elo1 better), 'H0' (accept + candidate is <= elo0, i.e. not a real improvement), or None + (inconclusive, keep playing).""" + llr = self.llr() + if llr >= self.lb: + return "H1" + if llr <= self.la: + return "H0" + return None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("head_engine", help="path to the baseline/head typhoon binary") + ap.add_argument("candidate_engine", help="path to the candidate typhoon binary") + ap.add_argument("--pgn", required=True) + ap.add_argument("--games", type=int, default=500, + help="total games; rounded down to an even number " + "since each opening is played as a pair " + "(candidate White / candidate Black). With " + "--sprt, this is a safety cap instead of a " + "fixed target -- the run stops as soon as the " + "SPRT concludes, or here if it never does; " + "pass something generous (e.g. 20000) when " + "using --sprt.") + ap.add_argument("--workers", type=int, default=8) + ap.add_argument("--sprt", action="store_true", + help="stop as soon as a Sequential Probability " + "Ratio Test concludes, instead of always " + "playing exactly --games games. See the " + "'how many games' discussion in CLAUDE.md/" + "session history for why this matters: a " + "fixed-N test either overpays for a large, " + "obvious effect or is simply too small to " + "resolve a marginal one -- SPRT adapts to " + "whichever it turns out to be.") + ap.add_argument("--elo0", type=float, default=0.0, + help="SPRT H0: candidate is at most this many Elo " + "better than baseline (the 'no real " + "improvement' hypothesis)") + ap.add_argument("--elo1", type=float, default=5.0, + help="SPRT H1: candidate is at least this many Elo " + "better than baseline (the 'real improvement' " + "hypothesis)") + ap.add_argument("--alpha", type=float, default=0.05, + help="SPRT false-positive rate (probability of " + "accepting H1 when H0 is actually true)") + ap.add_argument("--beta", type=float, default=0.05, + help="SPRT false-negative rate (probability of " + "accepting H0 when H1 is actually true)") + tc = ap.add_mutually_exclusive_group() + tc.add_argument("--sd", type=int, default=None, + help="fixed search depth per move for both sides " + "(default when neither --sd nor --st is given; " + "avoids machine-load noise -- see CLAUDE.md's " + "sd-over-st guidance). NOTE: --sd used to " + "default to 8 even when --st was passed, which " + "silently ignored --st entirely -- fixed so " + "the two are actually mutually exclusive now.") + tc.add_argument("--st", type=int, default=None, + help="seconds per move per side instead of fixed depth") + ap.add_argument("--max-plies", type=int, default=200) + ap.add_argument("--scratch", default="/usr/local/tmp/typhoon_match") + ap.add_argument("--log", default="match_history.log") + ap.add_argument("--pgn-out", default="match_games.pgn", + help="every played game is appended here as PGN") + args = ap.parse_args() + if args.sd is None and args.st is None: + args.sd = 8 + + head_engine = str(Path(args.head_engine).resolve()) + candidate_engine = str(Path(args.candidate_engine).resolve()) + + n_pairs = args.games // 2 + openings = sample_openings(args.pgn, n_pairs) + n_games = len(openings) * 2 + + jobs = [] + for fen in openings: + for candidate_is_white in (True, False): + i = len(jobs) + white_engine = candidate_engine if candidate_is_white else head_engine + black_engine = head_engine if candidate_is_white else candidate_engine + jobs.append((i, fen, candidate_is_white, white_engine, black_engine)) + + candidate_points = 0.0 + wins = draws = losses = 0 + sprt = Sprt(args.elo0, args.elo1, args.alpha, args.beta) if args.sprt else None + sprt_decision = None + candidate_stats = EngineStats() + head_stats = EngineStats() + + def run(job): + i, fen, candidate_is_white, white_engine, black_engine = job + game_start = time.monotonic() + result_white, moves, white_stats, black_stats = play_one_game( + white_engine, black_engine, f"{args.scratch}/g{i}", fen, + depth=args.sd, time_control_sec=args.st, + max_plies=args.max_plies, + ) + game_elapsed = time.monotonic() - game_start + candidate_result = result_white if candidate_is_white else (1.0 - result_white) + white_name = "candidate" if candidate_is_white else "baseline" + black_name = "baseline" if candidate_is_white else "candidate" + game, _ = build_pgn_game(fen, moves, result_white, white_name, black_name) + this_candidate_stats = white_stats if candidate_is_white else black_stats + this_head_stats = black_stats if candidate_is_white else white_stats + return (i, candidate_result, game, game_elapsed, + this_candidate_stats, this_head_stats) + + game_durations = [] + job_iter = iter(jobs) + done = 0 + # Games are processed in *completion* order (as_completed/wait), not + # submission order -- and decisive games plausibly finish faster than + # grindy draws/losses do (a winning side often wraps up well before + # --max-plies; a losing/drawing side tends to run long, sometimes all + # the way to the ply cap). That means naively printing a running + # candidate_points/done as results arrive is a biased mid-run + # estimator: wins arrive disproportionately early, so it reads high + # at first and erodes toward the true value as slower non-win games + # trickle in -- a real, reproducible artifact, not noise, and not + # evidence the actual match methodology is broken (the *final* + # score, summed over the complete set, is order-independent). Fixed + # by buffering out-of-order completions and only advancing the + # printed "score so far" through jobs in their original submission + # order (pending[]/next_report_idx below), so the progress line + # reflects an honest prefix rather than a completion-order-biased + # sample. + pending = {} + next_report_idx = 0 + reported_points = 0.0 + reported_count = 0 + + with ThreadPoolExecutor(max_workers=args.workers) as pool, \ + open(args.log, "a") as log, open(args.pgn_out, "a") as pgn_out: + in_flight = set() + for _ in range(args.workers): + job = next(job_iter, None) + if job is None: + break + in_flight.add(pool.submit(run, job)) + + while in_flight: + finished, in_flight = wait(in_flight, return_when=FIRST_COMPLETED) + for fut in finished: + (i, candidate_result, game, game_elapsed, + this_candidate_stats, this_head_stats) = fut.result() + done += 1 + candidate_points += candidate_result + game_durations.append(game_elapsed) + candidate_stats.merge(this_candidate_stats) + head_stats.merge(this_head_stats) + print(game, file=pgn_out, end="\n\n") + pgn_out.flush() + if candidate_result == 1.0: + wins += 1 + elif candidate_result == 0.0: + losses += 1 + else: + draws += 1 + if sprt is not None: + sprt.update(candidate_result) + log.write(f"game {i}: candidate_result={candidate_result} " + f"elapsed={game_elapsed:.1f}s\n") + log.flush() + + pending[i] = candidate_result + while next_report_idx in pending: + reported_points += pending.pop(next_report_idx) + reported_count += 1 + next_report_idx += 1 + + if done % 10 == 0: + avg_game_sec = sum(game_durations) / len(game_durations) + eta_sec = (avg_game_sec * + max(len(jobs) - done, 0)) / args.workers + sprt_note = "" + if sprt is not None: + sprt_note = (f" llr={sprt.llr():+.2f} " + f"(H0<={sprt.la:.2f} " + f"H1>={sprt.lb:.2f})") + print(f" {done} games played " + f"(score, in submission order through game " + f"{next_report_idx}: " + f"{reported_points/max(reported_count,1):.3f}) " + f"avg={avg_game_sec:.1f}s/game " + f"ETA={eta_sec/60:.1f}min{sprt_note}", + file=sys.stderr) + + if sprt is not None and sprt_decision is None: + sprt_decision = sprt.decision() + if sprt_decision is not None: + print(f" SPRT concluded: {sprt_decision} after " + f"{sprt.n} games (llr={sprt.llr():+.3f})", + file=sys.stderr) + + if sprt_decision is None: + while len(in_flight) < args.workers: + job = next(job_iter, None) + if job is None: + break + in_flight.add(pool.submit(run, job)) + # else: stop topping up, just let whatever's already + # in_flight finish naturally -- no wasted work beyond games + # already launched before the decision landed. + + n_games = done + score = candidate_points / n_games if n_games else 0.5 + elo, err = elo_diff(score, n_games) + # 95% normal-approximation CI directly on the score (not the elo + # transform, which distorts near 0/1) -- this is what the gate + # should key off of: a point estimate >= 0.5 is not evidence of + # improvement on its own when search is non-deterministic (see + # MP=1 multithreaded search noise observed even with identical + # DNA on both sides), only a lower bound that clears 0.5 is. + se_score = math.sqrt(score * (1 - score) / max(n_games, 1)) + lower95 = score - 1.96 * se_score + sprt_str = "" + if sprt is not None: + outcome = sprt_decision or "INCONCLUSIVE" + sprt_str = (f" SPRT={outcome} LLR={sprt.llr():+.3f} " + f"BOUNDS=[{sprt.la:.3f},{sprt.lb:.3f}] " + f"ELO0={args.elo0:.1f} ELO1={args.elo1:.1f}") + print(f"CANDIDATE_SCORE={score:.4f} GAMES={n_games} " + f"WINS={wins} DRAWS={draws} LOSSES={losses} " + f"ELO={elo:+.1f}+/-{err:.1f} LOWER95={lower95:.4f}{sprt_str}") + print(f"CANDIDATE_STATS: {candidate_stats.summary()}") + print(f"HEAD_STATS: {head_stats.summary()}") + + +if __name__ == "__main__": + main() |