1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#!/usr/bin/env bash
#
# Gated self-play match against a baseline binary. Historically this
# always compared the live working-tree build against head_reference/
# -- generalized (2026-09-14) to take both binary paths as args so two
# arbitrary builds (e.g. a file-storm-only worktree build vs. the
# current working tree, to isolate one change at a time when several
# have landed together -- see king_safety_data/HANDOFF.md's session
# notes on not measuring two eval changes in one match) can be compared
# without hand-rolling a match_play.py invocation each time.
#
# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label]
#
# head_binary Baseline engine. Defaults to head_reference/typhoon
# (the usual "is this better than last checkpoint"
# question).
# candidate_binary Candidate engine. Defaults to a fresh copy of the
# live working-tree build (../typhoon) into
# /tmp/typhoon/typhoon_match_play_candidate -- pass
# an explicit path (already built) to skip that
# copy/freeze step entirely, e.g. when comparing
# two prebuilt worktree binaries.
# label Distinguishes scratch dir/log/pgn-out so multiple
# matches can run concurrently without colliding.
# Defaults to "match".
#
# ...and any further arguments are passed straight through to
# match_play.py, so a one-off flag no longer needs this file edited.
#
# Three env vars override the baked-in defaults (2026-10-03), so the
# common knobs are reachable without touching the invocation below:
#
# TC_ARGS time control. Default: "--st 4"
# SPRT_ARGS hypothesis test. Default: "--sprt --elo0 0 --elo1 5"
# GAMES game cap. Default: 10000
#
# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5"
# cannot resolve the effect sizes this project's LMR work actually
# produces. At the observed draw rate the standard error in Elo is about
# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000
# games (~53h here) -- and every LMR match run to date has instead
# wandered around llr 0 for 2,000+ games and been abandoned, which is
# exactly what a true effect sitting BETWEEN the two hypotheses looks
# like. For a change you would only ship at >= 15 Elo, use
# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours
# instead of never.
#
# Examples:
# test_vs_head.sh # old default behavior
# test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight
#
# # Faster-rejecting bounds for a change only worth shipping at >=15 Elo:
# SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh
#
# # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the
# # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in
# # THIS engine, which bounds what any depth-buying search change
# # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity
# # rather than testing a hypothesis, and ELO=+/- prints either way.
# # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo
# # of precision already separates "a ply is worth 25" from "worth 65".
# SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \
# test_vs_head.sh ../../head_reference/typhoon \
# ../../head_reference/typhoon timeodds
set -e
HEAD="${1:-../../head_reference/typhoon}"
CANDIDATE_ARG="${2:-}"
LABEL="${3:-match}"
# Note the `-` (not `:-`) for the three env overrides below: an
# explicitly EMPTY value must be honored, not replaced by the default.
# SPRT_ARGS="" is the documented way to run without a hypothesis test
# (e.g. the time-odds calibration above), and `:-` would silently put
# --sprt back.
TC_ARGS="${TC_ARGS---st 4}"
SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}"
GAMES="${GAMES-10000}"
# Drop the (up to) three positional args so "$@" holds only extra
# pass-through flags for match_play.py. This MUST come after the three
# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above:
# an earlier version of this shifted first and then tested "$2" to
# decide whether to rebuild, which (a) made every no-extra-args run
# take the rebuild branch and silently clobber ../typhoon with a
# different make profile, and (b) made the first pass-through FLAG VALUE
# land in "$2" and get used as the candidate BINARY PATH -- producing a
# baffling "PermissionError: ... Permission denied: <scratch dir>" from
# Popen trying to execute a directory.
for _ in 1 2 3; do
if [ $# -gt 0 ]; then shift; fi
done
if [ -n "$CANDIDATE_ARG" ]; then
CANDIDATE="$CANDIDATE_ARG"
else
cd ..
gmake clean
gmake release MP=1 PERF_COUNTERS=1
cd eval_tune
CANDIDATE=/tmp/typhoon/typhoon_match_play_candidate
cp -f ../typhoon ${CANDIDATE}
fi
python3 ./match_play.py \
${HEAD} \
${CANDIDATE} \
--pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \
--games ${GAMES} \
--workers 18 \
--post \
${TC_ARGS} \
${SPRT_ARGS} \
--scratch /usr/local/tmp/typhoon_match_${LABEL} \
--log ${LABEL}.log \
"$@"
|
