1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
|
#!/usr/bin/env bash
#
# Gated self-play match against a baseline binary. Historically this
# always compared the live working-tree build against head_reference/
# -- generalized (2026-09-14) to take both binary paths as args so two
# arbitrary builds (e.g. a file-storm-only worktree build vs. the
# current working tree, to isolate one change at a time when several
# have landed together -- see king_safety_data/HANDOFF.md's session
# notes on not measuring two eval changes in one match) can be compared
# without hand-rolling a match_play.py invocation each time.
#
# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label]
#
# head_binary Baseline engine. Defaults to head_reference/typhoon
# (the usual "is this better than last checkpoint"
# question).
# candidate_binary Candidate engine. Defaults to a fresh copy of the
# live working-tree build (../typhoon) into
# /tmp/typhoon/typhoon_match_play_candidate -- pass
# an explicit path (already built) to skip that
# copy/freeze step entirely, e.g. when comparing
# two prebuilt worktree binaries.
# label Distinguishes scratch dir/log/pgn-out so multiple
# matches can run concurrently without colliding.
# Defaults to "match".
#
# ...and any further arguments are passed straight through to
# match_play.py, so a one-off flag no longer needs this file edited.
#
# Three env vars override the baked-in defaults (2026-10-03), so the
# common knobs are reachable without touching the invocation below:
#
# TC_ARGS time control. Default: "--st 4"
# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5"
# GAMES game cap. Default: 10000
#
# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5"
# cannot resolve the effect sizes this project's LMR work actually
# produces. At the observed draw rate the standard error in Elo is about
# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000
# games (~53h here) -- and every LMR match run to date has instead
# wandered around llr 0 for 2,000+ games and been abandoned, which is
# exactly what a true effect sitting BETWEEN the two hypotheses looks
# like. For a change you would only ship at >= 15 Elo, use
# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours
# instead of never.
#
# Examples:
# test_vs_head.sh # old default behavior
# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight
#
# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo:
# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh
#
# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the
# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in
# # THIS engine, which bounds what any depth-buying search change
# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity
# # rather than testing a hypothesis, and ELO=+/- prints either way.
# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo
# # of precision already separates "a ply is worth 25" from "worth 65".
# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \
# test_vs_head.sh ../../head_reference/typhoon \
# ../../head_reference/typhoon timeodds
#
# NNUE_WEIGHTS: path to a weights file (as produced by annotate_pgns/
# export_nnue_weights.py). When set, the CANDIDATE binary is wrapped
# in a thin exec shim that always injects "--nnueweights $NNUE_WEIGHTS"
# ahead of whatever args match_play.py's Engine class passes it --
# that's the only change needed, since Engine spawns the given path
# directly via Popen (no shell), so a shebang script in its place works
# unmodified. HEAD is untouched, so it keeps running the classic HCE --
# this is how you get an NNUE-candidate-vs-HCE-reference match without
# patching match_play.py's Engine (which currently passes identical
# engine_args to both sides):
#
# NNUE_WEIGHTS=/path/to/net.nnue test_vs_head.sh
#
# NNUE_PROBE_WEIGHTS: path to a probe weights file (as produced by
# annotate_pgns's probe-fitting script, "NPRB" format -- see docs/
# NNUE.md's "Lazy-evaluation tiering plan"). Requires NNUE_WEIGHTS to
# also be set (the probe gate only does anything once NNUE eval itself
# is enabled) -- adds "--nnueprobeweights $NNUE_PROBE_WEIGHTS" to the
# same wrapper, enabling the lazy-eval accumulator probe gate at
# startup (main.c's --nnueprobeweights sets g_fUseNnueLazyProbe=TRUE
# automatically, same convention as --nnueweights):
#
# NNUE_WEIGHTS=/path/to/net.nnue \
# NNUE_PROBE_WEIGHTS=/path/to/probe.bin \
# test_vs_head.sh
set -e
# Only consume LEADING args that don't look like a flag as the three
# positionals (head/candidate/label) -- 2026-10-19 fix: a bare
# ./nnue_vs_head.sh --sd 10 used to land "--sd" in HEAD and "10" in
# CANDIDATE_ARG (since the old code unconditionally took "$1"/"$2"/"$3"
# regardless of content), the exact same footgun class the comment
# below already describes once biting the --shift-then-test-"$2"
# version of this script. A flag-looking first arg now means "no
# positionals given at all, everything is pass-through" instead.
POSITIONALS=()
while [ $# -gt 0 ] && [ "${#POSITIONALS[@]}" -lt 3 ]; do
case "$1" in
-*) break ;;
*) POSITIONALS+=("$1"); shift ;;
esac
done
HEAD="${POSITIONALS[0]:-../../head_reference/typhoon}"
CANDIDATE_ARG="${POSITIONALS[1]:-}"
LABEL="${POSITIONALS[2]:-match}"
# Note the `-` (not `:-`) for the three env overrides below: an
# explicitly EMPTY value must be honored, not replaced by the default.
# SPRT_ARGS="" is the documented way to run without a hypothesis test
# (e.g. the time-odds calibration above), and `:-` would silently put
# --sprt back.
TC_ARGS="${TC_ARGS---st 4}"
SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}"
GAMES="${GAMES-10000}"
# If the caller already passed --sd or --st through as an extra
# argument (e.g. ./nnue_vs_head.sh --sd 10), don't ALSO inject the
# baked-in TC_ARGS ("--st 4" by default) -- match_play.py's --sd/--st
# are mutually exclusive, so the two would collide
# ("error: argument --st: not allowed with argument --sd") even though
# the caller only asked for one of them. Pass-through wins; TC_ARGS is
# only the fallback when the caller didn't specify either.
for arg in "$@"; do
case "$arg" in
--sd|--st) TC_ARGS="" ;;
esac
done
if [ -n "$CANDIDATE_ARG" ]; then
CANDIDATE="$CANDIDATE_ARG"
else
cd ..
gmake clean
gmake release MP=1 PERF_COUNTERS=1
cd eval_tune
CANDIDATE=/tmp/typhoon/typhoon_match_play_candidate
cp -f ../typhoon_nnue ${CANDIDATE}
# typhoon_nnue's own TYPHOON_BIN/NNUE_WEIGHTS defaults are computed
# relative to ITS OWN location ("next to this script" -- see its
# header comment) -- correct when it's the real src/typhoon_nnue,
# but wrong once copied to /tmp/typhoon/ above: it would then look
# for the engine binary at /tmp/typhoon/typhoon and the weights
# file under /tmp/annotate_pgns/..., neither of which exist, and
# exit 1 immediately on every single spawn (2026-10-19: this is
# what produced hundreds of instant "BrokenPipeError" game aborts
# the first time this ran -- the copied wrapper was exiting before
# ever speaking the xboard protocol). match_play.py's Popen
# inherits this script's environment, so exporting the correct
# ABSOLUTE paths here -- computed from this script's own known-good
# location, not the relocated copy's -- fixes it without needing
# to touch typhoon_nnue itself. Only fill in NNUE_WEIGHTS if the
# caller didn't already set one explicitly.
export TYPHOON_BIN="$(cd .. && pwd)/typhoon"
: "${NNUE_WEIGHTS:=$(cd ../../annotate_pgns && pwd)/runs/sf98k/nnue_ckpt/nnue_weights.nnue}"
export NNUE_WEIGHTS
fi
if [ -n "${NNUE_WEIGHTS:-}" ]; then
NNUE_WRAPPER=/tmp/typhoon/typhoon_match_play_candidate_nnue
NNUE_FLAGS="--nnueweights \"$NNUE_WEIGHTS\""
if [ -n "${NNUE_PROBE_WEIGHTS:-}" ]; then
NNUE_FLAGS="$NNUE_FLAGS --nnueprobeweights \"$NNUE_PROBE_WEIGHTS\""
fi
# NNUE_LAZY_MARGIN / NNUE_SUPERLAZY_MARGIN: 2026-10-16 looser-
# margin experiment (see docs/NNUE.md and command.c's "nnue
# lazymargin"/"nnue superlazyon"). NNUE_LAZY_MARGIN overrides every
# material bucket's lazy-probe margin with one flat cp value
# instead of the calibrated p98 table; NNUE_SUPERLAZY_MARGIN turns
# on the even-cheaper material-only pre-filter (off by default)
# with the given flat cp margin. Both require NNUE_WEIGHTS; the
# lazy one further requires NNUE_PROBE_WEIGHTS to do anything.
if [ -n "${NNUE_LAZY_MARGIN:-}" ]; then
NNUE_FLAGS="$NNUE_FLAGS --nnuelazymargin $NNUE_LAZY_MARGIN"
fi
if [ -n "${NNUE_SUPERLAZY_MARGIN:-}" ]; then
NNUE_FLAGS="$NNUE_FLAGS --nnuesuperlazymargin $NNUE_SUPERLAZY_MARGIN"
fi
cat > "$NNUE_WRAPPER" <<WRAPPER
#!/usr/bin/env bash
exec "$CANDIDATE" $NNUE_FLAGS "\$@"
WRAPPER
chmod +x "$NNUE_WRAPPER"
CANDIDATE="$NNUE_WRAPPER"
fi
# NNUE_CANDIDATE_NAME: PGN White/Black identity recorded for the
# candidate engine, forwarded to match_play.py's --candidate-name
# (2026-10-16, see that flag's own comment). Defaults to a name that
# flags "don't trust this as training data" whenever NNUE is in play,
# since self_play_games.pgn is reused later to train the NNUE net
# itself and a differently-calibrated evaluator's scores shouldn't be
# mixed in as labels without this being mechanically filterable.
# Override explicitly if a given NNUE config's scores are actually
# meant to be trusted.
NAME_ARGS=""
if [ -n "${NNUE_WEIGHTS:-}" ]; then
NNUE_CANDIDATE_NAME="${NNUE_CANDIDATE_NAME:-candidate_nnue}"
NAME_ARGS="--candidate-name $NNUE_CANDIDATE_NAME"
fi
python3 ./match_play.py \
${HEAD} \
${CANDIDATE} \
--pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \
--games ${GAMES} \
--workers 18 \
--post \
${TC_ARGS} \
${SPRT_ARGS} \
${NAME_ARGS} \
--scratch /usr/local/tmp/typhoon_match_${LABEL} \
--log ${LABEL}.log \
"$@"
|
