diff options
Diffstat (limited to 'src/eval_tune/match_play.py')
| -rwxr-xr-x | src/eval_tune/match_play.py | 29 |
1 files changed, 20 insertions, 9 deletions
diff --git a/src/eval_tune/match_play.py b/src/eval_tune/match_play.py index 6891964..9591330 100755 --- a/src/eval_tune/match_play.py +++ b/src/eval_tune/match_play.py @@ -367,6 +367,18 @@ def elo_to_score(elo): return 1.0 / (1.0 + 10.0 ** (-elo / 400.0)) +def sprt_bar(llr, la, lb, width=9): + """Render an ASCII gauge of where `llr` sits between the H0 (`la`) and + H1 (`lb`) SPRT bounds, e.g. '|---------|V---------|' with V marking + the rounded llr position.""" + frac = 0.5 if lb == la else (llr - la) / (lb - la) + frac = min(max(frac, 0.0), 1.0) + slot = round(frac * (2 * width)) + bar = "|" + "-" * width + "|" + "-" * width + "|" + idx = slot + 1 + return bar[:idx] + "V" + bar[idx:] + + class Sprt: """Sequential Probability Ratio Test for engine-vs-engine gating, same formulation fishtest/cutechess-cli use for exactly this problem: two @@ -602,17 +614,16 @@ def main(): avg_game_sec = sum(game_durations) / len(game_durations) eta_sec = (avg_game_sec * max(len(jobs) - done, 0)) / args.workers + score = candidate_points / done sprt_note = "" if sprt is not None: - sprt_note = (f" llr={sprt.llr():+.2f} " - f"(H0<={sprt.la:.2f} " - f"H1>={sprt.lb:.2f})") - print(f" {done} games played " - f"(score, in submission order through game " - f"{next_report_idx}: " - f"{reported_points/max(reported_count,1):.3f}) " - f"avg={avg_game_sec:.1f}s/game " - f"ETA={eta_sec/60:.1f}min{sprt_note}", + sprt_note = (f": llr={sprt.llr():+.2f}, " + f"H0={sprt.la:+.2f}" + f"{sprt_bar(sprt.llr(), sprt.la, sprt.lb)}" + f"H1={sprt.lb:+.2f}") + print(f" {done} games: -{losses} ={draws} +{wins} " + f"({avg_game_sec:.1f}s avg, eta={eta_sec/60:.0f}min, " + f"score={score:.3f}){sprt_note}", file=sys.stderr) if sprt is not None and sprt_decision is None: |