1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
#!/usr/bin/env python3
"""Cross-check eval_tune/tune_eval_dna.py's Python static_exchange_eval()
(a classic swap-list SEE that admittedly ignores pins/x-rays/
discoveries -- see its docstring) against the engine's own real SEE(),
on real positions from an annotate.py run.
This has never been done before: static_exchange_eval() has been used
by the eval-tuning pipeline's quiet-position filter since that
existed, but nothing has ever actually verified it agrees with the
engine it's trying to approximate. Worth doing once, as reusable
tooling, since this pipeline (annotate -> quiet-filter -> train) is
going to be re-run many times as the project continues.
Mechanism: the engine already has a bulk oracle for exactly this,
`seescores <filename>` (command.c) -- takes a file of "setboard <fen>"
lines (the same format shard_NN.script already uses) and prints
"SEESCORES <fen> | <san> | <see-value>" for every legal capture in
every position in the file, no search involved. Sample N FENs from an
existing positions.jsonl, write them to a scratch "setboard"-lines
file, run `seescores` on it, and diff every (fen, move) pair's engine
SEE value against the Python one.
Usage:
python3 check_see_against_engine.py --positions runs/sd10_twic_2M/positions.jsonl \
--sample 2000
"""
import argparse
import json
import os
import random
import re
import subprocess
import sys
import tempfile
from collections import Counter
from pathlib import Path
_EVAL_TUNE_DIR = Path(__file__).resolve().parent.parent / "src" / "eval_tune"
sys.path.insert(0, str(_EVAL_TUNE_DIR))
import chess
# See quiet_filter.py's comment: dna_diff (imported by tune_eval_dna)
# reads "../eval.c" relative to CWD at import time, not to its own file.
_cwd = os.getcwd()
os.chdir(_EVAL_TUNE_DIR)
try:
from tune_eval_dna import static_exchange_eval
finally:
os.chdir(_cwd)
SEESCORES_RE = re.compile(r"^SEESCORES (\S+(?: \S+)*?) \| (\S+) \| (-?\d+)$")
def sample_fens(positions_path, n, seed):
fens = []
with open(positions_path) as f:
for line in f:
rec = json.loads(line)
if rec.get("exact"):
continue
fens.append(rec["fen"])
random.Random(seed).shuffle(fens)
return fens[:n]
def run_seescores(engine_path, fens, scratch_dir):
script_path = Path(scratch_dir) / "see_check.script"
out_path = Path(scratch_dir) / "see_check.out"
with open(script_path, "w") as f:
for fen in fens:
f.write(f"setboard {fen}\n")
cmd = [str(engine_path), "--batch", "--command", f"seescores {script_path}"]
with open(out_path, "w") as out:
subprocess.run(cmd, stdout=out, stderr=subprocess.STDOUT, check=True)
return out_path
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--positions", required=True,
help="positions.jsonl to sample FENs from")
ap.add_argument("--sample", type=int, default=2000)
ap.add_argument("--seed", type=int, default=0)
ap.add_argument("--engine", default="../src/typhoon")
args = ap.parse_args()
engine_path = Path(args.engine).resolve()
if not engine_path.exists():
sys.exit(f"engine not found: {engine_path}")
print(f"Sampling {args.sample} non-exact positions from {args.positions} ...")
fens = sample_fens(args.positions, args.sample, args.seed)
print(f" got {len(fens)} FENs")
with tempfile.TemporaryDirectory() as scratch:
print("Running engine seescores ...")
out_path = run_seescores(engine_path, fens, scratch)
n_compared = 0
n_exact_match = 0
n_sign_match = 0
n_sign_mismatch = 0
worst = [] # (abs diff, fen, san, engine_val, python_val)
diff_histogram = Counter()
board_cache = {}
with open(out_path) as f:
for line in f:
m = SEESCORES_RE.match(line.rstrip("\n"))
if not m:
continue
fen, san, engine_val = m.groups()
engine_val = int(engine_val)
board = board_cache.get(fen)
if board is None:
board = chess.Board(fen)
board_cache[fen] = board
try:
move = board.parse_san(san)
except ValueError:
continue
python_val = static_exchange_eval(board, move)
n_compared += 1
diff = engine_val - python_val
if diff == 0:
n_exact_match += 1
bucket = "0" if diff == 0 else (
"<=50" if abs(diff) <= 50 else
"<=200" if abs(diff) <= 200 else ">200")
diff_histogram[bucket] += 1
engine_sign = (engine_val > 0) - (engine_val < 0)
python_sign = (python_val > 0) - (python_val < 0)
if engine_sign == python_sign:
n_sign_match += 1
else:
n_sign_mismatch += 1
worst.append((abs(diff), fen, san, engine_val, python_val))
worst.sort(reverse=True)
print(f"\nCompared {n_compared} (fen, capture) pairs")
print(f" exact match: {n_exact_match} ({n_exact_match/max(n_compared,1):.1%})")
print(f" sign match: {n_sign_match} ({n_sign_match/max(n_compared,1):.1%})")
print(f" sign MISmatch:{n_sign_mismatch} ({n_sign_mismatch/max(n_compared,1):.1%})")
print(f" |diff| histogram: {dict(diff_histogram)}")
if worst:
print(f"\nWorst {min(10, len(worst))} disagreements "
f"(|engine - python|, fen, san, engine_see, python_see):")
for d, fen, san, ev, pv in worst[:10]:
print(f" {d:5d} {fen} {san} engine={ev} python={pv}")
if __name__ == "__main__":
main()
|
