1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
#!/usr/bin/env bash
#
# Gated self-play match against a baseline binary. Historically this
# always compared the live working-tree build against head_reference/
# -- generalized (2026-09-14) to take both binary paths as args so two
# arbitrary builds (e.g. a file-storm-only worktree build vs. the
# current working tree, to isolate one change at a time when several
# have landed together -- see king_safety_data/HANDOFF.md's session
# notes on not measuring two eval changes in one match) can be compared
# without hand-rolling a match_play.py invocation each time.
#
# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label]
#
# head_binary Baseline engine. Defaults to head_reference/typhoon
# (the usual "is this better than last checkpoint"
# question).
# candidate_binary Candidate engine. Defaults to a fresh copy of the
# live working-tree build (../typhoon) into
# /tmp/typhoon/typhoon_match_play_candidate -- pass
# an explicit path (already built) to skip that
# copy/freeze step entirely, e.g. when comparing
# two prebuilt worktree binaries.
# label Distinguishes scratch dir/log/pgn-out so multiple
# matches can run concurrently without colliding.
# Defaults to "match".
#
# ...and any further arguments are passed straight through to
# match_play.py, so a one-off flag no longer needs this file edited.
#
# Three env vars override the baked-in defaults (2026-10-03), so the
# common knobs are reachable without touching the invocation below:
#
# TC_ARGS time control. Default: "--st 4"
# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5"
# GAMES game cap. Default: 10000
#
# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5"
# cannot resolve the effect sizes this project's LMR work actually
# produces. At the observed draw rate the standard error in Elo is about
# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000
# games (~53h here) -- and every LMR match run to date has instead
# wandered around llr 0 for 2,000+ games and been abandoned, which is
# exactly what a true effect sitting BETWEEN the two hypotheses looks
# like. For a change you would only ship at >= 15 Elo, use
# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours
# instead of never.
#
# Examples:
# test_vs_head.sh # old default behavior
# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight
#
# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo:
# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh
#
# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the
# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in
# # THIS engine, which bounds what any depth-buying search change
# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity
# # rather than testing a hypothesis, and ELO=+/- prints either way.
# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo
# # of precision already separates "a ply is worth 25" from "worth 65".
# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \
# test_vs_head.sh ../../head_reference/typhoon \
# ../../head_reference/typhoon timeodds
#
# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/
# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped
# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS"
# ahead of whatever args match_play.py's Engine class passes it --
# that's the only change needed, since Engine spawns the given path
# directly via Popen (no shell), so a shebang script in its place works
# unmodified. HEAD is untouched, so it keeps running the classic HCE --
# this is how you get an NNUE-candidate-vs-HCE-reference match without
# patching match_play.py's Engine (which currently passes identical
# engine_args to both sides):
#
# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh
set -e
HEAD="${1:-../../head_reference/typhoon}"
CANDIDATE_ARG="${2:-}"
LABEL="${3:-match}"
# Note the `-` (not `:-`) for the three env overrides below: an
# explicitly EMPTY value must be honored, not replaced by the default.
# SPRT_ARGS="" is the documented way to run without a hypothesis test
# (e.g. the time-odds calibration above), and `:-` would silently put
# --sprt back.
TC_ARGS="${TC_ARGS---st 4}"
SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}"
GAMES="${GAMES-10000}"
# Drop the (up to) three positional args so "$@" holds only extra
# pass-through flags for match_play.py. This MUST come after the three
# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above:
# an earlier version of this shifted first and then tested "$2" to
# decide whether to rebuild, which (a) made every no-extra-args run
# take the rebuild branch and silently clobber ../typhoon with a
# different make profile, and (b) made the first pass-through FLAG VALUE
# land in "$2" and get used as the candidate BINARY PATH -- producing a
# baffling "PermissionError: ... Permission denied: <scratch dir>" from
# Popen trying to execute a directory.
for _ in 1 2 3; do
if [ $# -gt 0 ]; then shift; fi
done
if [ -n "$CANDIDATE_ARG" ]; then
CANDIDATE="$CANDIDATE_ARG"
else
cd ..
gmake clean
gmake release MP=1 PERF_COUNTERS=1
cd eval_tune
CANDIDATE=/tmp/typhoon/typhoon_match_play_candidate
cp -f ../typhoon ${CANDIDATE}
fi
if [ -n "${NNUE_WEIGHTS:-}" ]; then
NNUE_WRAPPER=/tmp/typhoon/typhoon_match_play_candidate_nnue
cat > "$NNUE_WRAPPER" <<WRAPPER
#!/usr/bin/env bash
exec "$CANDIDATE" --nnueweights "$NNUE_WEIGHTS" "\$@"
WRAPPER
chmod +x "$NNUE_WRAPPER"
CANDIDATE="$NNUE_WRAPPER"
fi
python3 ./match_play.py \
${HEAD} \
${CANDIDATE} \
--pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \
--games ${GAMES} \
--workers 18 \
--post \
${TC_ARGS} \
${SPRT_ARGS} \
--scratch /usr/local/tmp/typhoon_match_${LABEL} \
--log ${LABEL}.log \
"$@"
|
