1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
#!/usr/bin/env bash
set -e
LOGDIR="/tmp/typhoon/manual"
HEAD_LOGDIR="../head_reference/logs"
TESTS="../tests"
# Suite names default to the three curated suites (see CLAUDE.md's
# head_reference protocol). Override on the command line to run other
# suites, e.g. ./run_tests.sh ecm_ringers ecm_lmr_delta
SUITES=("$@")
if [ ${#SUITES[@]} -eq 0 ]; then
SUITES=(ecm_ringers ecm_confident_quick ecm_hard_quick)
fi
# sd (fixed depth) is the right choice when judging whether a pruning/
# ordering change makes the tree smaller or larger for the same search
# effort; sn (fixed node budget) is the right choice when judging how many
# positions solve within a fixed cost. Override via SD/SN env vars --
# this applies to every suite passed on the command line.
#
# ecm_late.ep_ (see its header comment) is a different shape of suite --
# curated for positions where the *correct* move is only confirmed well
# into the search, specifically so avg-solution-time/total-nodecount from
# the stats block below can be read as an extensions/search-quality
# metric ("does the engine confirm these tactics sooner, in aggregate")
# rather than a solve-count gate. It's sn-only (no sd sweep): the
# discovery data behind it was measured on an 8-thread box (avg ~13 ply /
# ~70M total nodes across all 8 threads) and a fixed *depth* on 1 thread,
# with no cutoff-sharing across threads to shrink the tree, can blow up
# unpredictably on the sharpest of these positions -- a fixed *node*
# budget bounds wall-clock regardless. 15M nodes (~12s/position at this
# box's ~1.3Mnps single-thread rate, ~5 min for all 25 positions) is
# picked to stay in "quick ad hoc iteration" territory while still being
# in the right ballpark of the per-thread node share implied by the
# 8-thread discovery data -- not a rigorously derived number, adjust via
# SN if it's cutting off too many positions before they confirm.
SD="${SD:-10}"
SN="${SN:-5000000}"
SN_LABEL="${SN_LABEL:-5m}"
ECM_LATE_SN=15000000
ECM_LATE_SN_LABEL=15m
suite_sn_only() {
[ "$1" = "ecm_late" ]
}
suite_sn() {
if [ "$1" = "ecm_late" ]; then echo "$ECM_LATE_SN"; else echo "$SN"; fi
}
suite_sn_label() {
if [ "$1" = "ecm_late" ]; then echo "$ECM_LATE_SN_LABEL"; else echo "$SN_LABEL"; fi
}
/bin/rm -rf "${LOGDIR}"
mkdir -p "${LOGDIR}"
echo -n "Launching test suites (${SUITES[*]}) in parallel... "
_start_time=$(date +%s)
for suite in "${SUITES[@]}"; do
suite_sn=$(suite_sn "$suite")
if ! suite_sn_only "$suite"; then
./typhoon \
--cpus 1 \
--hash 256m \
--logfile "$LOGDIR/sd${SD}_${suite}.log" \
--batch \
--command "force; book name /nonexistent.book.bin; sd ${SD}; script $TESTS/${suite}.ep_" \
> "$LOGDIR/sd${SD}_${suite}.out" 2>&1 &
fi
./typhoon \
--cpus 1 \
--hash 256m \
--logfile "$LOGDIR/sn${suite_sn}_${suite}.log" \
--batch \
--command "force; book name /nonexistent.book.bin; sn ${suite_sn}; script $TESTS/${suite}.ep_" \
> "$LOGDIR/sn${suite_sn}_${suite}.out" 2>&1 &
done
wait
_elapsed=$(( $(date +%s) - _start_time ))
echo "DONE! (${_elapsed}s)"
echo
print_stats_table() {
# $1 = head .out file, $2 = candidate .out file
# Extracts the "Final statistics" block (correct solutions .. script time)
# from each and prints a HEAD/CANDIDATE table with a numeric diff column.
local headfile="$1" candfile="$2"
awk -F: '
NR==FNR {
n++; lbl[n]=$1; val1[n]=$2; next
}
{
m++; val2[m]=$2
}
END {
printf(" %-22s %-35s %-35s\n", "", "HEAD", "CANDIDATE")
for (i = 1; i <= n; i++) {
label = lbl[i]; gsub(/^ +| +$/, "", label)
v1 = val1[i]; gsub(/^ +| +$/, "", v1)
v2 = (i <= m ? val2[i] : ""); gsub(/^ +| +$/, "", v2)
if (label == "leftover") {
gsub(/^[0-9.]+% alpha, +/, "", v1)
gsub(/^[0-9.]+% alpha, +/, "", v2)
}
n1 = v1 + 0; n2 = v2 + 0
diff = n2 - n1
if (diff == 0) {
dstr = ""
} else if (diff == int(diff)) {
dstr = sprintf(" (%+d)", diff)
} else {
dstr = sprintf(" (%+.2f)", diff)
}
printf(" %-22s %-35s %-35s\n", label ":", v1, v2 dstr)
}
}
' "$headfile" "$candfile"
}
for suite in "${SUITES[@]}"; do
suite_sn=$(suite_sn "$suite")
suite_sn_label=$(suite_sn_label "$suite")
if ! suite_sn_only "$suite"; then
echo "----[ $suite sd=${SD} ]------------------------------------------------------------------"
print_stats_table \
<(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sd${SD}_${suite}.out") \
<(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sd${SD}_${suite}.out")
echo
fi
echo "----[ $suite sn=${suite_sn} ]------------------------------------------------------------------"
print_stats_table \
<(sed -n '/correct solutions/,/script time/p' "$HEAD_LOGDIR/sn${suite_sn_label}_${suite}.out") \
<(sed -n '/correct solutions/,/script time/p' "$LOGDIR/sn${suite_sn}_${suite}.out")
echo
done
echo
echo "NOTE: The logfiles for these runs are in /tmp/typhoon/manual and will be"
echo " overwritten by the next invocation of this script; save now them if"
echo " you want them!"
|
