#!/usr/bin/env bash
#
# Gated self-play match against a baseline binary. Historically this
# always compared the live working-tree build against head_reference/
# -- generalized (2026-09-14) to take both binary paths as args so two
# arbitrary builds (e.g. a file-storm-only worktree build vs. the
# current working tree, to isolate one change at a time when several
# have landed together -- see king_safety_data/HANDOFF.md's session
# notes on not measuring two eval changes in one match) can be compared
# without hand-rolling a match_play.py invocation each time.
#
# Usage: test_vs_head.sh [head_binary] [candidate_binary] [label]
#
#   head_binary       Baseline engine. Defaults to head_reference/typhoon
#                      (the usual "is this better than last checkpoint"
#                      question).
#   candidate_binary   Candidate engine. Defaults to a fresh copy of the
#                      live working-tree build (../typhoon) into
#                      /tmp/typhoon/typhoon_match_play_candidate -- pass
#                      an explicit path (already built) to skip that
#                      copy/freeze step entirely, e.g. when comparing
#                      two prebuilt worktree binaries.
#   label              Distinguishes scratch dir/log/pgn-out so multiple
#                      matches can run concurrently without colliding.
#                      Defaults to "match".
#
#   ...and any further arguments are passed straight through to
#   match_play.py, so a one-off flag no longer needs this file edited.
#
# Three env vars override the baked-in defaults (2026-10-03), so the
# common knobs are reachable without touching the invocation below:
#
#   TC_ARGS      time control.      Default: "--st 4"
#   SPRT_ARGS    hypothesis test.   Default: "--sprt --elo0 0 --elo1 5"
#   GAMES        game cap.          Default: 10000
#
# Why these became overridable: the baked-in "--sprt --elo0 0 --elo1 5"
# cannot resolve the effect sizes this project's LMR work actually
# produces. At the observed draw rate the standard error in Elo is about
# 249/sqrt(n), so separating 0 from 5 Elo at even 2 sigma needs ~10,000
# games (~53h here) -- and every LMR match run to date has instead
# wandered around llr 0 for 2,000+ games and been abandoned, which is
# exactly what a true effect sitting BETWEEN the two hypotheses looks
# like. For a change you would only ship at >= 15 Elo, use
# SPRT_ARGS="--sprt --elo0 0 --elo1 15": it rejects a +2 effect in hours
# instead of never.
#
# Examples:
#   test_vs_head.sh                                                    # old default behavior
#   test_vs_head.sh /tmp/typhoon/filestorm_only /tmp/typhoon/candidate queenprox_backflight
#
#   # Faster-rejecting bounds for a change only worth shipping at >=15 Elo:
#   SPRT_ARGS="--sprt --elo0 0 --elo1 15" test_vs_head.sh
#
#   # TIME-ODDS CALIBRATION: same binary both sides, candidate gets 2x the
#   # time. Measures what ~1.6 extra plies (at ebf 3.6) is worth in Elo in
#   # THIS engine, which bounds what any depth-buying search change
#   # (LMR/EFP/LMP) can ever deliver. No SPRT -- this measures a quantity
#   # rather than testing a hypothesis, and ELO=+/- prints either way.
#   # ~600-1000 games is plenty: the expected effect is large, so +/-10 Elo
#   # of precision already separates "a ply is worth 25" from "worth 65".
#   SPRT_ARGS="" TC_ARGS="--st 4 --odds 2" GAMES=1000 \
#       test_vs_head.sh ../../head_reference/typhoon \
#                        ../../head_reference/typhoon timeodds

set -e

HEAD="${1:-../../head_reference/typhoon}"
CANDIDATE_ARG="${2:-}"
LABEL="${3:-match}"
# Note the `-` (not `:-`) for the three env overrides below: an
# explicitly EMPTY value must be honored, not replaced by the default.
# SPRT_ARGS="" is the documented way to run without a hypothesis test
# (e.g. the time-odds calibration above), and `:-` would silently put
# --sprt back.
TC_ARGS="${TC_ARGS---st 4}"
SPRT_ARGS="${SPRT_ARGS---sprt --elo0 0 --elo1 5}"
GAMES="${GAMES-10000}"

# Drop the (up to) three positional args so "$@" holds only extra
# pass-through flags for match_play.py. This MUST come after the three
# positionals have been captured into HEAD/CANDIDATE_ARG/LABEL above:
# an earlier version of this shifted first and then tested "$2" to
# decide whether to rebuild, which (a) made every no-extra-args run
# take the rebuild branch and silently clobber ../typhoon with a
# different make profile, and (b) made the first pass-through FLAG VALUE
# land in "$2" and get used as the candidate BINARY PATH -- producing a
# baffling "PermissionError: ... Permission denied: <scratch dir>" from
# Popen trying to execute a directory.
for _ in 1 2 3; do
    if [ $# -gt 0 ]; then shift; fi
done

if [ -n "$CANDIDATE_ARG" ]; then
    CANDIDATE="$CANDIDATE_ARG"
else
    cd ..
    gmake clean
    gmake release MP=1 PERF_COUNTERS=1
    cd eval_tune
    CANDIDATE=/tmp/typhoon/typhoon_match_play_candidate
    cp -f ../typhoon ${CANDIDATE}
fi

python3 ./match_play.py \
        ${HEAD} \
        ${CANDIDATE} \
        --pgn ../../pgn/twic_filtered.pgn ../../pgn/lichess*.pgn \
        --games ${GAMES} \
        --workers 18 \
        --post \
        ${TC_ARGS} \
        ${SPRT_ARGS} \
        --scratch /usr/local/tmp/typhoon_match_${LABEL} \
        --log ${LABEL}.log \
        "$@"