path: root/src/eval_tune/match_play.py

#!/usr/bin/env python3
"""
Head-to-head match: head binary vs. candidate binary, N games, alternating
colors, opening positions sampled from a filtered TWIC pool for variety.

Two separate compiled typhoon binaries (built from different commits/
working trees), not one binary with two loaded `evaldna` files -- the
DNA-diffing design this replaced was built for the Texel auto-tuning
pipeline, which this project has since moved away from in favor of
hand-tuned eval constants baked directly into eval.c. Comparing two
binaries means a real build is required for each side before running
this (there's no live DNA swap), but it's the right model now: what's
actually being compared is two different eval.c/search.c source trees,
not two parameter files loaded into an otherwise-identical process.

The authoritative board is kept in python-chess; each engine is treated
as stateless per move (force; setboard <fen>; go) rather than fed
"usermove", mirroring the go/force turn-taking already verified to work
in tune_eval_dna.py's play_selfplay_game.

Usage:
    python3 match_play.py <head_engine_path> <candidate_engine_path> \\
        --pgn ../../pgn/twic_filtered.pgn --games 500 --workers 4 --st 1

    Add --sprt (with --elo0/--elo1/--alpha/--beta) to stop as soon as a
    Sequential Probability Ratio Test concludes instead of always
    playing exactly --games games -- see the Sprt class docstring for
    why a fixed game count is usually the wrong tool for "are these two
    engines really different in strength".

Prints:
    CANDIDATE_SCORE=<0..1> GAMES=<n> WINS=<a> DRAWS=<b> LOSSES=<c> ELO=<+/-x.x>
        LOWER95=<x> [SPRT=... LLR=... BOUNDS=... ELO0=... ELO1=...]
    CANDIDATE_STATS: nps=... ebf=... first_move_beta=...% nodes=... plies=...
    HEAD_STATS:      nps=... ebf=... first_move_beta=...% nodes=... plies=...
for the orchestrator to parse. Speed/tree-shape stats (EngineStats) are
pulled from the same per-move PostMoveSearchReport block every "go"
already prints, not a separate benchmark -- see EngineStats' docstring.
"""
import argparse
import hashlib
import io
import math
import random
import re
import sys
import time
from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED
from pathlib import Path

import chess
import chess.pgn

sys.path.insert(0, str(Path(__file__).parent))
from tune_eval_dna import Engine  # noqa: E402

_MOVE_RE = re.compile(r"^move (\S+)")
# PostMoveSearchReport (root.c) prints these AFTER the "move" line, not
# before -- confirmed against root.c (Trace("move %s\n", ...) at line
# ~1367/1369, PostMoveSearchReport(ctx) called at ~1384, strictly after)
# and against real engine output. A per-move stats block trails every
# move, not just script-run suite summaries -- see EBF/first-move-beta
# discussion in session history.
_SEARCHED_RE = re.compile(
    r"^Searched for\s+([\d.]+) seconds, saw (\d+) nodes "
    r"\((\d+) qnodes\) \(\s*([\d.]+) nps\)\.")
_DEPTH_RE = re.compile(r"^tellothers d(\d+),")
_FIRST_MOVE_BETA_RE = re.compile(
    r"^First move beta cutoff rate was\s+([\d.]+) percent\.")
_STATS_TRAILING_LINES = 20  # bound on how far past "move" to keep reading


def sample_openings(pgn_path, n, min_ply=8, max_ply=20, seed=0,
                     cache_dir="opening_cache"):
    """Pick n random (roughly) opening positions out of a pgn pool by
    seeking to random byte offsets and parsing just the one game found
    there, rather than sequentially chess.pgn.read_game()-ing from the
    start -- on a several-hundred-MB pool, replaying every game up to
    the sample point (even while mostly skipping them) means real I/O
    and SAN-parsing cost that scales with how far into the file the
    first few accepted samples happen to land, which was observed
    taking minutes for as few as 2 requested openings. Cached like
    tune_eval_dna.py's position_cache, keyed on pgn identity + params,
    since repeated cycle.sh runs against the same pool shouldn't pay
    this more than once."""
    st = Path(pgn_path).stat()
    key = hashlib.sha256(
        f"{pgn_path}|{st.st_mtime}|{st.st_size}|{n}|{min_ply}|{max_ply}|{seed}"
        .encode()
    ).hexdigest()[:16]
    cache_path = Path(cache_dir) / f"openings_{key}.txt"
    if cache_path.exists():
        openings = cache_path.read_text().splitlines()
        print(f"  loaded {len(openings)} openings from cache {cache_path}",
              file=sys.stderr)
        return openings

    rng = random.Random(seed)
    file_size = st.st_size
    openings = []
    attempts = 0
    max_attempts = n * 50 + 200  # generous; each attempt is one seek+one game parse
    with open(pgn_path, "rb") as fb:
        while len(openings) < n and attempts < max_attempts:
            attempts += 1
            offset = rng.randint(0, max(file_size - 1, 0))
            fb.seek(offset)
            fb.readline()  # discard partial line at the seek point
            # scan forward to the start of the next game's header block
            start = fb.tell()
            line = fb.readline()
            while line and not line.startswith(b"[Event "):
                start = fb.tell()
                line = fb.readline()
            if not line:
                continue  # landed past the last game in the file; retry elsewhere
            # read forward to the START of the game AFTER this one (or EOF)
            fb.seek(start)
            fb.readline()  # consume this game's own "[Event " line
            end = fb.tell()
            line = fb.readline()
            while line and not line.startswith(b"[Event "):
                end = fb.tell()
                line = fb.readline()
            fb.seek(start)
            raw = fb.read(end - start).decode("utf-8", errors="replace")
            game = chess.pgn.read_game(io.StringIO(raw))
            if game is None:
                continue
            moves = list(game.mainline_moves())
            if len(moves) < min_ply:
                continue
            board = game.board()
            ply = rng.randint(min_ply, min(max_ply, len(moves)))
            for mv in moves[:ply]:
                board.push(mv)
            openings.append(board.fen())

    if len(openings) < n:
        print(f"warning: only found {len(openings)} openings "
              f"(wanted {n}) in {pgn_path}", file=sys.stderr)

    cache_path.parent.mkdir(parents=True, exist_ok=True)
    cache_path.write_text("\n".join(openings) + "\n" if openings else "")
    return openings


def play_one_game(white_engine_path, black_engine_path, scratch_dir, fen,
                   depth=None, time_control_sec=None, max_plies=200):
    """Returns (result, moves, white_stats, black_stats) where result is
    1.0/0.5/0.0 from White's perspective, moves is the list of UCI moves
    actually played (for PGN reconstruction), and white_stats/black_stats
    are EngineStats instances covering that side's moves in this game."""
    moves = []
    white_stats = EngineStats()
    black_stats = EngineStats()
    white_last_fmb = black_last_fmb = None
    # Default Engine() spawns with NO --hash/--cpus at all, which means
    # a tiny 65536-entry hash table (main.c's memset-zero default) --
    # far smaller than the 256m used everywhere else in this project's
    # own testing (test.sh). A small hash causes much worse move
    # ordering/more re-search in complex positions -- pass explicit
    # options here so gate-match search timing is representative
    # rather than an artifact of an unset default.
    # --logfile matters here, not just cosmetically: the default is a
    # shared "typhoon.log" in cwd (main.c's memset-zero default),
    # opened via BackupFile()-then-fopen(wb+) at startup and fflush()ed
    # on every Log()/Bug() call. With N concurrent engine processes (2
    # per game x --workers games) all doing that against the SAME
    # file, you get real rename races and lock contention -- observed
    # as unexplained match slowdown. "-" disables file logging
    # entirely (main.c: skips BackupFile/fopen altogether) -- fine
    # here since match_play.py never reads engine logs (moves come off
    # stdout, games get their own match_games.pgn).
    engine_args = ["--hash", "256m", "--cpus", "1", "--logfile", "-"]
    with Engine(white_engine_path, f"{scratch_dir}/w", engine_args) as we, \
         Engine(black_engine_path, f"{scratch_dir}/b", engine_args) as be:
        # Disable book and pondering.  Set time control.  Resign lost positions.
        tc_cmd = f"sd {depth}" if depth is not None else f"st {time_control_sec}"
        for command in ( "book name /none",
                         "set ThinkOnOpponentsTime false",
                         "set ResignThreshold -1000",
                         tc_cmd):
            we._send(command)
            be._send(command)

        board = chess.Board(fen)
        for _ in range(max_plies):
            if board.is_game_over(claim_draw=True):
                break
            white_to_move = board.turn == chess.WHITE
            mover = we if white_to_move else be
            mover_stats = white_stats if white_to_move else black_stats
            mover._send("force")
            mover._send(f"setboard {board.fen()}")
            mover._send("go")
            move_uci = None
            for _ in range(400):
                line = mover.stdout.readline()
                if not line:
                    break
                if line.startswith("tellics resign"):
                    # Engine resigned instead of moving -- root.c prints
                    # this and returns without ever printing "move ...",
                    # so stop reading now rather than blocking forever.
                    break
                m = _MOVE_RE.match(line)
                if m:
                    move_uci = m.group(1)
                    break
            if move_uci is None:
                # side to move had nothing to play -- treat as loss for mover
                result = 0.0 if white_to_move else 1.0
                return result, moves, white_stats, black_stats

            # PostMoveSearchReport's stats block trails the "move" line
            # (confirmed against root.c and real output), not the other
            # way around -- keep reading a bounded number of further
            # lines to pick it up before moving on to the next ply.
            ply_nodes = ply_time = ply_depth = ply_fmb = None
            for _ in range(_STATS_TRAILING_LINES):
                line = mover.stdout.readline()
                if not line:
                    break
                sm = _SEARCHED_RE.match(line)
                if sm:
                    ply_time = float(sm.group(1))
                    ply_nodes = int(sm.group(2))
                    continue
                dm = _DEPTH_RE.match(line)
                if dm:
                    ply_depth = int(dm.group(1))
                    continue
                fm = _FIRST_MOVE_BETA_RE.match(line)
                if fm:
                    ply_fmb = float(fm.group(1))
                    break  # last line of the block we care about
            if ply_nodes is not None and ply_time is not None:
                mover_stats.add_ply(ply_nodes, ply_time, ply_depth)
            if ply_fmb is not None:
                if white_to_move:
                    white_last_fmb = ply_fmb
                else:
                    black_last_fmb = ply_fmb

            try:
                move = chess.Move.from_uci(move_uci)
                if move not in board.legal_moves:
                    result = 0.0 if white_to_move else 1.0
                    return result, moves, white_stats, black_stats
            except ValueError:
                result = 0.0 if white_to_move else 1.0
                return result, moves, white_stats, black_stats
            board.push(move)
            moves.append(move_uci)

        white_stats.add_game_first_move_beta(white_last_fmb, white_stats.plies)
        black_stats.add_game_first_move_beta(black_last_fmb, black_stats.plies)

        outcome = board.outcome(claim_draw=True)
        if outcome is None or outcome.winner is None:
            result = 0.5
        else:
            result = 1.0 if outcome.winner == chess.WHITE else 0.0
        return result, moves, white_stats, black_stats


def build_pgn_game(start_fen, moves, result_white, white_name, black_name):
    board = chess.Board(start_fen)
    game = chess.pgn.Game()
    game.headers["Event"] = "typhoon DNA gate match"
    game.headers["White"] = white_name
    game.headers["Black"] = black_name
    game.headers["FEN"] = start_fen
    game.setup(board)
    result_str = {1.0: "1-0", 0.0: "0-1", 0.5: "1/2-1/2"}[result_white]
    game.headers["Result"] = result_str
    node = game
    for move_uci in moves:
        node = node.add_variation(chess.Move.from_uci(move_uci))
    return game, result_str


def elo_diff(score, n):
    """Rough Elo estimate + a 95%-ish error bar from a win rate, treating
    games as independent Bernoulli-ish trials on the score. Good enough
    as a signal, not a substitute for a real SPRT."""
    score = min(max(score, 1e-6), 1 - 1e-6)
    elo = -400 * math.log10(1 / score - 1)
    # standard error on the score, propagated through the logit
    se_score = math.sqrt(score * (1 - score) / max(n, 1))
    se_elo = (400 / math.log(10)) * se_score / (score * (1 - score))
    return elo, 1.96 * se_elo


class EngineStats:
    """Per-engine (not per-color -- candidate/baseline swap sides every
    other game) search-quality accumulator: speed and tree shape, pulled
    from the same PostMoveSearchReport block every move already prints
    (see _SEARCHED_RE/_DEPTH_RE/_FIRST_MOVE_BETA_RE), not a separate
    benchmark. EBF is computed the same way script.c's suite runs do
    (nodes**(1/depth) per move, see head_reference notes on why this can
    diverge from first-move-beta-cutoff-rate when pruning/aspiration-
    window re-searches inflate node counts independent of true tree
    shape), then averaged across all moves an engine played. First-move
    beta cutoff rate is read as a running total inside a single engine
    process's lifetime (root.c never resets those counters mid-process),
    and since play_one_game spawns a fresh process per game, the value
    from an engine's last move of a game is already that whole game's
    rate -- summed here across games as a plies-weighted average so a
    long game doesn't count the same as a short one.
    """

    def __init__(self):
        self.total_nodes = 0
        self.total_time = 0.0
        self.plies = 0
        self.ebf_sum = 0.0
        self.ebf_count = 0
        self._fmb_weighted_sum = 0.0
        self._fmb_weight = 0

    def add_ply(self, nodes, elapsed, depth):
        self.total_nodes += nodes
        self.total_time += elapsed
        self.plies += 1
        if depth and depth > 0 and nodes > 0:
            self.ebf_sum += nodes ** (1.0 / depth)
            self.ebf_count += 1

    def add_game_first_move_beta(self, pct, plies_in_game):
        if pct is not None and plies_in_game > 0:
            self._fmb_weighted_sum += pct * plies_in_game
            self._fmb_weight += plies_in_game

    def merge(self, other):
        self.total_nodes += other.total_nodes
        self.total_time += other.total_time
        self.plies += other.plies
        self.ebf_sum += other.ebf_sum
        self.ebf_count += other.ebf_count
        self._fmb_weighted_sum += other._fmb_weighted_sum
        self._fmb_weight += other._fmb_weight

    @property
    def nps(self):
        return self.total_nodes / self.total_time if self.total_time else 0.0

    @property
    def avg_ebf(self):
        return self.ebf_sum / self.ebf_count if self.ebf_count else 0.0

    @property
    def avg_first_move_beta(self):
        return (self._fmb_weighted_sum / self._fmb_weight
                if self._fmb_weight else 0.0)

    def summary(self):
        return (f"nps={self.nps:.0f} ebf={self.avg_ebf:.3f} "
                f"first_move_beta={self.avg_first_move_beta:.1f}% "
                f"nodes={self.total_nodes} plies={self.plies}")


def elo_to_score(elo):
    """Expected per-game score (0..1) for a side rated `elo` points above
    an opponent, via the standard logistic Elo model."""
    return 1.0 / (1.0 + 10.0 ** (-elo / 400.0))


class Sprt:
    """Sequential Probability Ratio Test for engine-vs-engine gating, same
    formulation fishtest/cutechess-cli use for exactly this problem: two
    Elo hypotheses (H0: true strength gap is elo0 or worse, H1: elo1 or
    better), tested via the log-likelihood ratio of a normal
    approximation to the per-game trinomial (W/D/L) score, with variance
    re-estimated from the running W/D/L mix as games accumulate.

    Unlike a fixed-N test (this script's original LOWER95 gate), this
    stops as soon as the evidence is conclusive in either direction --
    a handful of games if the true effect is large, potentially tens of
    thousands if it's genuinely marginal -- rather than committing to
    one game count up front regardless of the actual effect size.

    See CLAUDE.md / the "how many games" discussion this was built from:
    for a 95%/80%-power fixed-N test, resolving a 5 Elo gap needs
    ~35-40k games, 10 Elo needs ~9-10k, 20-30 Elo needs ~1-2.5k. SPRT
    doesn't remove that fundamental cost, it just avoids overpaying for
    it when the true effect is far from the boundary either way.
    """

    def __init__(self, elo0, elo1, alpha=0.05, beta=0.05):
        self.elo0 = elo0
        self.elo1 = elo1
        self.s0 = elo_to_score(elo0)
        self.s1 = elo_to_score(elo1)
        self.la = math.log(beta / (1 - alpha))
        self.lb = math.log((1 - beta) / alpha)
        self.wins = self.draws = self.losses = 0

    def update(self, result):
        """result: 1.0 (candidate win), 0.5 (draw), or 0.0 (candidate loss)."""
        if result == 1.0:
            self.wins += 1
        elif result == 0.0:
            self.losses += 1
        else:
            self.draws += 1

    @property
    def n(self):
        return self.wins + self.draws + self.losses

    def llr(self):
        n = self.n
        if n == 0:
            return 0.0
        mean = (self.wins + 0.5 * self.draws) / n
        var = (self.wins * (1 - mean) ** 2 +
               self.draws * (0.5 - mean) ** 2 +
               self.losses * (0 - mean) ** 2) / n
        if var <= 0:
            # every game so far had the identical result -- LLR is only
            # defined once there's some spread; treat as inconclusive.
            return 0.0
        return n * (self.s1 - self.s0) * (mean - (self.s0 + self.s1) / 2) / var

    def decision(self):
        """Returns 'H1' (accept candidate is >= elo1 better), 'H0' (accept
        candidate is <= elo0, i.e. not a real improvement), or None
        (inconclusive, keep playing)."""
        llr = self.llr()
        if llr >= self.lb:
            return "H1"
        if llr <= self.la:
            return "H0"
        return None


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("head_engine", help="path to the baseline/head typhoon binary")
    ap.add_argument("candidate_engine", help="path to the candidate typhoon binary")
    ap.add_argument("--pgn", required=True)
    ap.add_argument("--games", type=int, default=500,
                     help="total games; rounded down to an even number "
                          "since each opening is played as a pair "
                          "(candidate White / candidate Black). With "
                          "--sprt, this is a safety cap instead of a "
                          "fixed target -- the run stops as soon as the "
                          "SPRT concludes, or here if it never does; "
                          "pass something generous (e.g. 20000) when "
                          "using --sprt.")
    ap.add_argument("--workers", type=int, default=8)
    ap.add_argument("--sprt", action="store_true",
                     help="stop as soon as a Sequential Probability "
                          "Ratio Test concludes, instead of always "
                          "playing exactly --games games. See the "
                          "'how many games' discussion in CLAUDE.md/"
                          "session history for why this matters: a "
                          "fixed-N test either overpays for a large, "
                          "obvious effect or is simply too small to "
                          "resolve a marginal one -- SPRT adapts to "
                          "whichever it turns out to be.")
    ap.add_argument("--elo0", type=float, default=0.0,
                     help="SPRT H0: candidate is at most this many Elo "
                          "better than baseline (the 'no real "
                          "improvement' hypothesis)")
    ap.add_argument("--elo1", type=float, default=5.0,
                     help="SPRT H1: candidate is at least this many Elo "
                          "better than baseline (the 'real improvement' "
                          "hypothesis)")
    ap.add_argument("--alpha", type=float, default=0.05,
                     help="SPRT false-positive rate (probability of "
                          "accepting H1 when H0 is actually true)")
    ap.add_argument("--beta", type=float, default=0.05,
                     help="SPRT false-negative rate (probability of "
                          "accepting H0 when H1 is actually true)")
    tc = ap.add_mutually_exclusive_group()
    tc.add_argument("--sd", type=int, default=None,
                     help="fixed search depth per move for both sides "
                          "(default when neither --sd nor --st is given; "
                          "avoids machine-load noise -- see CLAUDE.md's "
                          "sd-over-st guidance). NOTE: --sd used to "
                          "default to 8 even when --st was passed, which "
                          "silently ignored --st entirely -- fixed so "
                          "the two are actually mutually exclusive now.")
    tc.add_argument("--st", type=int, default=None,
                     help="seconds per move per side instead of fixed depth")
    ap.add_argument("--max-plies", type=int, default=200)
    ap.add_argument("--scratch", default="/usr/local/tmp/typhoon_match")
    ap.add_argument("--log", default="match_history.log")
    ap.add_argument("--pgn-out", default="match_games.pgn",
                     help="every played game is appended here as PGN")
    args = ap.parse_args()
    if args.sd is None and args.st is None:
        args.sd = 8

    head_engine = str(Path(args.head_engine).resolve())
    candidate_engine = str(Path(args.candidate_engine).resolve())

    n_pairs = args.games // 2
    openings = sample_openings(args.pgn, n_pairs)
    n_games = len(openings) * 2

    jobs = []
    for fen in openings:
        for candidate_is_white in (True, False):
            i = len(jobs)
            white_engine = candidate_engine if candidate_is_white else head_engine
            black_engine = head_engine if candidate_is_white else candidate_engine
            jobs.append((i, fen, candidate_is_white, white_engine, black_engine))

    candidate_points = 0.0
    wins = draws = losses = 0
    sprt = Sprt(args.elo0, args.elo1, args.alpha, args.beta) if args.sprt else None
    sprt_decision = None
    candidate_stats = EngineStats()
    head_stats = EngineStats()

    def run(job):
        i, fen, candidate_is_white, white_engine, black_engine = job
        game_start = time.monotonic()
        result_white, moves, white_stats, black_stats = play_one_game(
            white_engine, black_engine, f"{args.scratch}/g{i}", fen,
            depth=args.sd, time_control_sec=args.st,
            max_plies=args.max_plies,
        )
        game_elapsed = time.monotonic() - game_start
        candidate_result = result_white if candidate_is_white else (1.0 - result_white)
        white_name = "candidate" if candidate_is_white else "baseline"
        black_name = "baseline" if candidate_is_white else "candidate"
        game, _ = build_pgn_game(fen, moves, result_white, white_name, black_name)
        this_candidate_stats = white_stats if candidate_is_white else black_stats
        this_head_stats = black_stats if candidate_is_white else white_stats
        return (i, candidate_result, game, game_elapsed,
                this_candidate_stats, this_head_stats)

    game_durations = []
    job_iter = iter(jobs)
    done = 0
    # Games are processed in *completion* order (as_completed/wait), not
    # submission order -- and decisive games plausibly finish faster than
    # grindy draws/losses do (a winning side often wraps up well before
    # --max-plies; a losing/drawing side tends to run long, sometimes all
    # the way to the ply cap). That means naively printing a running
    # candidate_points/done as results arrive is a biased mid-run
    # estimator: wins arrive disproportionately early, so it reads high
    # at first and erodes toward the true value as slower non-win games
    # trickle in -- a real, reproducible artifact, not noise, and not
    # evidence the actual match methodology is broken (the *final*
    # score, summed over the complete set, is order-independent). Fixed
    # by buffering out-of-order completions and only advancing the
    # printed "score so far" through jobs in their original submission
    # order (pending[]/next_report_idx below), so the progress line
    # reflects an honest prefix rather than a completion-order-biased
    # sample.
    pending = {}
    next_report_idx = 0
    reported_points = 0.0
    reported_count = 0

    with ThreadPoolExecutor(max_workers=args.workers) as pool, \
         open(args.log, "a") as log, open(args.pgn_out, "a") as pgn_out:
        in_flight = set()
        for _ in range(args.workers):
            job = next(job_iter, None)
            if job is None:
                break
            in_flight.add(pool.submit(run, job))

        while in_flight:
            finished, in_flight = wait(in_flight, return_when=FIRST_COMPLETED)
            for fut in finished:
                (i, candidate_result, game, game_elapsed,
                 this_candidate_stats, this_head_stats) = fut.result()
                done += 1
                candidate_points += candidate_result
                game_durations.append(game_elapsed)
                candidate_stats.merge(this_candidate_stats)
                head_stats.merge(this_head_stats)
                print(game, file=pgn_out, end="\n\n")
                pgn_out.flush()
                if candidate_result == 1.0:
                    wins += 1
                elif candidate_result == 0.0:
                    losses += 1
                else:
                    draws += 1
                if sprt is not None:
                    sprt.update(candidate_result)
                log.write(f"game {i}: candidate_result={candidate_result} "
                          f"elapsed={game_elapsed:.1f}s\n")
                log.flush()

                pending[i] = candidate_result
                while next_report_idx in pending:
                    reported_points += pending.pop(next_report_idx)
                    reported_count += 1
                    next_report_idx += 1

                if done % 10 == 0:
                    avg_game_sec = sum(game_durations) / len(game_durations)
                    eta_sec = (avg_game_sec *
                               max(len(jobs) - done, 0)) / args.workers
                    sprt_note = ""
                    if sprt is not None:
                        sprt_note = (f" llr={sprt.llr():+.2f} "
                                     f"(H0<={sprt.la:.2f} "
                                     f"H1>={sprt.lb:.2f})")
                    print(f"  {done} games played "
                          f"(score, in submission order through game "
                          f"{next_report_idx}: "
                          f"{reported_points/max(reported_count,1):.3f}) "
                          f"avg={avg_game_sec:.1f}s/game "
                          f"ETA={eta_sec/60:.1f}min{sprt_note}",
                          file=sys.stderr)

            if sprt is not None and sprt_decision is None:
                sprt_decision = sprt.decision()
                if sprt_decision is not None:
                    print(f"  SPRT concluded: {sprt_decision} after "
                          f"{sprt.n} games (llr={sprt.llr():+.3f})",
                          file=sys.stderr)

            if sprt_decision is None:
                while len(in_flight) < args.workers:
                    job = next(job_iter, None)
                    if job is None:
                        break
                    in_flight.add(pool.submit(run, job))
            # else: stop topping up, just let whatever's already
            # in_flight finish naturally -- no wasted work beyond games
            # already launched before the decision landed.

    n_games = done
    score = candidate_points / n_games if n_games else 0.5
    elo, err = elo_diff(score, n_games)
    # 95% normal-approximation CI directly on the score (not the elo
    # transform, which distorts near 0/1) -- this is what the gate
    # should key off of: a point estimate >= 0.5 is not evidence of
    # improvement on its own when search is non-deterministic (see
    # MP=1 multithreaded search noise observed even with identical
    # DNA on both sides), only a lower bound that clears 0.5 is.
    se_score = math.sqrt(score * (1 - score) / max(n_games, 1))
    lower95 = score - 1.96 * se_score
    sprt_str = ""
    if sprt is not None:
        outcome = sprt_decision or "INCONCLUSIVE"
        sprt_str = (f" SPRT={outcome} LLR={sprt.llr():+.3f} "
                    f"BOUNDS=[{sprt.la:.3f},{sprt.lb:.3f}] "
                    f"ELO0={args.elo0:.1f} ELO1={args.elo1:.1f}")
    print(f"CANDIDATE_SCORE={score:.4f} GAMES={n_games} "
          f"WINS={wins} DRAWS={draws} LOSSES={losses} "
          f"ELO={elo:+.1f}+/-{err:.1f} LOWER95={lower95:.4f}{sprt_str}")
    print(f"CANDIDATE_STATS: {candidate_stats.summary()}")
    print(f"HEAD_STATS:      {head_stats.summary()}")


if __name__ == "__main__":
    main()