#!/usr/bin/env bash # # Gated self-play match against a baseline binary. Historically this # always compared the live working-tree build against head_reference/ # -- generalized (2026-09-14) to take both binary paths as args so two # arbitrary builds (e.g. a file-storm-only worktree build vs. the # current working tree, to isolate one change at a time when several # have landed together -- see king_safety_data/HANDOFF.md's session # notes on not measuring two eval changes in one match) can be compared # without hand-rolling a match_play.py invocation each time. # # Usage: test_vs_head.sh [head_binary] [candidate_binary] [label] # # head_binary Baseline engine. Defaults to head_reference/typhoon # (the usual "is this better than last checkpoint" # question). # candidate_binary Candidate engine. Defaults to a fresh copy of the # live working-tree build (../typhoon) into # /tmp/typhoon/typhoon_match_play_candidate -- pass # an explicit path (already built) to skip that # copy/freeze step entirely, e.g. when comparing # two prebuilt worktree binaries. # label Distinguishes scratch dir/log/pgn-out so multiple # matches can run concurrently without colliding. # Defaults to "match". # # ...and any further arguments are passed straight through to # match_play.py, so a one-off flag no longer needs this file edited. # # Three env vars override the baked-in defaults (2026-10-03), so the # common knobs are reachable without touching the invocation below: # # TC_ARGS time control. Default: "--st 4" # SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5" # GAMES game cap. Default: 10000 # # Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5" # cannot resolve the effect sizes this project's LMR work actually # produces. At the observed draw rate the standard error in Elo is about # 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000 # games (~53h here) -- and every LMR match run to date has instead # wandered around llr 0 for 2,000+ games and been abandoned, which is # exactly what a true effect sitting BETWEEN the two hypotheses looks # like. For a change you would only ship at >= 15 Elo, use # SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours # instead of never. # # Examples: # test_vs_head.sh # old default behavior # test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight # # # Faster-rejecting bounds for a change only worth shipping at >=15 Elo: # SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh # # # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the # # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in # # THIS engine, which bounds what any depth-buying search change # # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity # # rather than testing a hypothesis, and ELO=+/- prints either way. # # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo # # of precision already separates "a ply is worth 25" from "worth 65". # SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \ # test_vs_head.sh ../../head_reference/typhoon \ # ../../head_reference/typhoon timeodds # # NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/ # export_nnue_weights.py). When set, the CANDIDATE binary is wrapped # in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS" # ahead of whatever args match_play.py's Engine class passes it -- # that's the only change needed, since Engine spawns the given path # directly via Popen (no shell), so a shebang script in its place works # unmodified. HEAD is untouched, so it keeps running the classic HCE -- # this is how you get an NNUE-candidate-vs-HCE-reference match without # patching match_play.py's Engine (which currently passes identical # engine_args to both sides): # # NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh # # NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by # annotate_pgns's probe-fitting script, "NPRB" format -- see docs/ # NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to # also be set (the probe gate only does anything once NNUE eval itself # is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the # same wrapper, enabling the lazy-eval accumulator probe gate at # startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE # automatically, same convention as --nnueweights): # # NNUE_WEIGHTS=/path/to/net.nnue \ # NNUE_PROBE_WEIGHTS=/path/to/probe.bin \ # test_vs_head.sh set -e # Only consume LEADING args that don't look like a flag as the three # positionals (head/candidate/label) -- 2026-10-19 fix: a bare # ./nnue_vs_head.sh --sd 10 used to land "--sd" in HEAD and "10" in # CANDIDATE_ARG (since the old code unconditionally took "$1"/"$2"/"$3" # regardless of content), the exact same footgun class the comment # below already describes once biting the --shift-then-test-"$2" # version of this script. A flag-looking first arg now means "no # positionals given at all, everything is pass-through" instead. POSITIONALS=() while [ $# -gt 0 ] && [ "${#POSITIONALS[@]}" -lt 3 ]; do case "$1" in -*) break ;; *) POSITIONALS+=("$1"); shift ;; esac done HEAD="${POSITIONALS[0]:-../../head_reference/typhoon}" CANDIDATE_ARG="${POSITIONALS[1]:-}" LABEL="${POSITIONALS[2]:-match}" # Note the `-` (not `:-`) for the three env overrides below: an # explicitly EMPTY value must be honored, not replaced by the default. # SPRT_ARGS="" is the documented way to run without a hypothesis test # (e.g. the time-odds calibration above), and `:-` would silently put # --sprt back. TC_ARGS="${TC_ARGS---st 4}" SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}" GAMES="${GAMES-10000}" # If the caller already passed --sd or --st through as an extra # argument (e.g. ./nnue_vs_head.sh --sd 10), don't ALSO inject the # baked-in TC_ARGS ("--st 4" by default) -- match_play.py's --sd/--st # are mutually exclusive, so the two would collide # ("error: argument --st: not allowed with argument --sd") even though # the caller only asked for one of them. Pass-through wins; TC_ARGS is # only the fallback when the caller didn't specify either. for arg in "$@"; do case "$arg" in --sd|--st) TC_ARGS="" ;; esac done if [ -n "$CANDIDATE_ARG" ]; then CANDIDATE="$CANDIDATE_ARG" else cd .. gmake clean gmake release MP=1 PERF_COUNTERS=1 cd eval_tune CANDIDATE=/tmp/typhoon/typhoon_match_play_candidate cp -f ../typhoon_nnue ${CANDIDATE} # typhoon_nnue's own TYPHOON_BIN/NNUE_WEIGHTS defaults are computed # relative to ITS OWN location ("next to this script" -- see its # header comment) -- correct when it's the real src/typhoon_nnue, # but wrong once copied to /tmp/typhoon/ above: it would then look # for the engine binary at /tmp/typhoon/typhoon and the weights # file under /tmp/annotate_pgns/..., neither of which exist, and # exit 1 immediately on every single spawn (2026-10-19: this is # what produced hundreds of instant "BrokenPipeError" game aborts # the first time this ran -- the copied wrapper was exiting before # ever speaking the xboard protocol). match_play.py's Popen # inherits this script's environment, so exporting the correct # ABSOLUTE paths here -- computed from this script's own known-good # location, not the relocated copy's -- fixes it without needing # to touch typhoon_nnue itself. Only fill in NNUE_WEIGHTS if the # caller didn't already set one explicitly. export TYPHOON_BIN="$(cd .. && pwd)/typhoon" : "${NNUE_WEIGHTS:=$(cd ../../annotate_pgns && pwd)/runs/sf98k/nnue_ckpt/nnue_weights.nnue}" export NNUE_WEIGHTS fi if [ -n "${NNUE_WEIGHTS:-}" ]; then NNUE_WRAPPER=/tmp/typhoon/typhoon_match_play_candidate_nnue NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\"" if [ -n "${NNUE_PROBE_WEIGHTS:-}" ]; then NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\"" fi # NNUE_LAZY_MARGIN / NNUE_SUPERLAZY_MARGIN: 2026-10-16 looser- # margin experiment (see docs/NNUE.md and command.c's "nnue # lazymargin"/"nnue superlazyon"). NNUE_LAZY_MARGIN overrides every # material bucket's lazy-probe margin with one flat cp value # instead of the calibrated p98 table; NNUE_SUPERLAZY_MARGIN turns # on the even-cheaper material-only pre-filter (off by default) # with the given flat cp margin. Both require NNUE_WEIGHTS; the # lazy one further requires NNUE_PROBE_WEIGHTS to do anything. if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN" fi if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN" fi cat > "$NNUE_WRAPPER" <