diff options
Diffstat (limited to 'src/eval_tune')
| -rwxr-xr-x | src/eval_tune/match_play.py | 29 | ||||
| -rwxr-xr-x | src/eval_tune/run_ecm.sh | 41 | ||||
| -rwxr-xr-x | src/eval_tune/test_vs_head.sh | 2 |
3 files changed, 21 insertions, 51 deletions
diff --git a/src/eval_tune/match_play.py b/src/eval_tune/match_play.py index 6891964..9591330 100755 --- a/src/eval_tune/match_play.py +++ b/src/eval_tune/match_play.py @@ -367,6 +367,18 @@ def elo_to_score(elo): return 1.0 / (1.0 + 10.0 ** (-elo / 400.0)) +def sprt_bar(llr, la, lb, width=9): + """Render an ASCII gauge of where `llr` sits between the H0 (`la`) and + H1 (`lb`) SPRT bounds, e.g. '|---------|V---------|' with V marking + the rounded llr position.""" + frac = 0.5 if lb == la else (llr - la) / (lb - la) + frac = min(max(frac, 0.0), 1.0) + slot = round(frac * (2 * width)) + bar = "|" + "-" * width + "|" + "-" * width + "|" + idx = slot + 1 + return bar[:idx] + "V" + bar[idx:] + + class Sprt: """Sequential Probability Ratio Test for engine-vs-engine gating, same formulation fishtest/cutechess-cli use for exactly this problem: two @@ -602,17 +614,16 @@ def main(): avg_game_sec = sum(game_durations) / len(game_durations) eta_sec = (avg_game_sec * max(len(jobs) - done, 0)) / args.workers + score = candidate_points / done sprt_note = "" if sprt is not None: - sprt_note = (f" llr={sprt.llr():+.2f} " - f"(H0<={sprt.la:.2f} " - f"H1>={sprt.lb:.2f})") - print(f" {done} games played " - f"(score, in submission order through game " - f"{next_report_idx}: " - f"{reported_points/max(reported_count,1):.3f}) " - f"avg={avg_game_sec:.1f}s/game " - f"ETA={eta_sec/60:.1f}min{sprt_note}", + sprt_note = (f": llr={sprt.llr():+.2f}, " + f"H0={sprt.la:+.2f}" + f"{sprt_bar(sprt.llr(), sprt.la, sprt.lb)}" + f"H1={sprt.lb:+.2f}") + print(f" {done} games: -{losses} ={draws} +{wins} " + f"({avg_game_sec:.1f}s avg, eta={eta_sec/60:.0f}min, " + f"score={score:.3f}){sprt_note}", file=sys.stderr) if sprt is not None and sprt_decision is None: diff --git a/src/eval_tune/run_ecm.sh b/src/eval_tune/run_ecm.sh deleted file mode 100755 index a6c1284..0000000 --- a/src/eval_tune/run_ecm.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -# Run the ECM tactical suite at a fixed search depth (not fixed time -- -# see CLAUDE.md: st introduces machine-load noise that sd avoids) and -# print just the "correct solutions" tally so callers can parse it. -# -# Usage: run_ecm.sh <label> [depth] -# Writes eval_tune/ecm_logs/<label>.log (full Trace() output) and -# appends one line to eval_tune/ecm_history.log. - -set -e -cd "$(dirname "$0")/.." # repo src/ root - -label="$1" -depth="${2:-11}" -if [ -z "$label" ]; then - echo "Usage: $0 <label> [depth]" >&2 - exit 1 -fi - -mkdir -p eval_tune/ecm_logs -logfile="eval_tune/ecm_logs/${label}.log" - -opts='--cpus 1 --hash 256m --egtbpath /zscratch/egtb' -# --batch exits non-zero on normal EOF ("Exhausted input in batch -# mode") -- that's expected, not a failure, so don't let set -e treat -# it as one; the real success/failure check is the grep below. -./typhoon ${opts} --logfile "$logfile" --batch \ - --command "sd ${depth}; script ../tests/ecm.ep_" || true - -correct=$(grep -o 'correct solutions : [0-9]*' "$logfile" | tail -1 | grep -o '[0-9]*$') -total=$(grep -o 'total problems : [0-9]*' "$logfile" | tail -1 | grep -o '[0-9]*$') - -if [ -z "$correct" ] || [ -z "$total" ]; then - echo "ERROR: couldn't parse solved count from $logfile" >&2 - exit 1 -fi - -echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) label=$label depth=$depth solved=${correct}/${total}" \ - >> eval_tune/ecm_history.log - -echo "${correct} ${total}" diff --git a/src/eval_tune/test_vs_head.sh b/src/eval_tune/test_vs_head.sh index c7c578e..d5d8006 100755 --- a/src/eval_tune/test_vs_head.sh +++ b/src/eval_tune/test_vs_head.sh @@ -3,7 +3,7 @@ python3 ./match_play.py ../../head_reference/typhoon ../typhoon \ --pgn ../../pgn/twic_filtered.pgn \ --games 20000 \ - --workers 20 \ + --workers 12 \ --st 1 \ --sprt --elo0 0 --elo1 5 \ --scratch /usr/local/tmp/typhoon_match_overnight \ |