Rewrite _EvalKing as a linear ring-scan and promote its terms to DNA-tunable constants; retune via mate-mining regression, ElasticNet/Ridge, and logistic-vs-material refits.

Splits _EvalKing's own king-location table from CountKingSafetyDefects' search-gate copy so future eval retunes can't silently perturb extension/reduction gate frequency. Bumps KING_BACK_FLIGHT_BONUS (+1 -> +3) and reorders/nudges the ring-scan sub-terms (minor/pawn/xray/undefended) to match relative rankings that replicated across three independent calibration methods, while leaving KING_QUEEN_PROXIMITY_DANGER and KING_MISSING_ONE_CASTLE_OPTION untouched pending further discussion.

Self-play (640 games) is roughly even (score 0.509, llr=+0.17, far from either SPRT bound) but consistently faster (nps candidate=1362140 vs head=1333682) at equal average depth -- landing as a wash/speed-neutral checkpoint, not a confirmed tactical improvement, for further tuning to build on.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

1 files changed, 160 insertions, 22 deletions

diff --git a/src/eval_tune/tune_eval_dna.py b/src/eval_tune/tune_eval_dna.py
index b3960f8..96796dc 100755
--- a/src/eval_tune/tune_eval_dna.py
+++ b/src/eval_tune/tune_eval_dna.py
@@ -67,8 +67,32 @@ class Engine:
         # flag (see its "--logfile matters here" comment). A caller
         # that actually wants a logfile can still pass extra_args
         # explicitly to override this default.
+        #
+        # --hash 1m (2026-09-15): main.c defaults to 256m with no
+        # --hash flag at all, and command.c's SetBoardCommand calls
+        # PreGameReset -> ClearHashTable on EVERY setboard, which is a
+        # full memset(g_pHashTable, 0, g_uHashTableSizeBytes) -- hash.c
+        # confirms this is unconditional, not incremental. Measured
+        # live: ~25ms per setboard+eval round trip with the 256m
+        # default, matching a 256MB memset almost exactly, and it's
+        # WORSE than linear across N parallel workers since memory
+        # bandwidth (unlike most CPU work) is a genuinely shared,
+        # finite resource -- 18 workers all memset-ing 256MB
+        # concurrently starved each other on the memory bus (measured
+        # 27 positions/s aggregate against an expected ~700/s if this
+        # were compute-bound and scaled cleanly). This class only ever
+        # scores one-off positions (no repeated-search benefit from a
+        # large TT anyway), so hashing is disabled outright rather than
+        # just shrunk -- 2026-09-15: "none" measured ~30% faster than
+        # "1m" (951/s vs 730/s single-process) with 0/200 score
+        # mismatches for both eval and qsearch against "1m" -- _ParseHashOption
+        # ("none"/"no" -> 0 entries) is a real, intentionally-supported
+        # engine option (hash.c's ClearHashTable guards on g_pHashTable
+        # != NULL, so a disabled table skips the memset for free), not
+        # an untested codepath, so no reason to prefer the smaller-but-
+        # nonzero table over this now that it's been measured safe.
         if extra_args is None:
-            extra_args = ["--logfile", "-"]
+            extra_args = ["--logfile", "-", "--hash", "none"]
         self.proc = subprocess.Popen(
             [path] + extra_args,
             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
@@ -83,6 +107,17 @@ class Engine:
         self._send("protover 2")
         self._drain_features()
         self._send("force")          # we set positions, engine doesn't play
+        # 2026-09-15: g_Options.fShouldPost defaults TRUE (main.c), so
+        # every setboard prints a full ASCII board diagram that this
+        # class's readers then have to skip past line-by-line -- real,
+        # measured cost at corpus scale (this is the exact bug
+        # CLAUDE.md already documents extract_king_features.py hitting
+        # and fixing; it just never got applied here). At ~9x the
+        # per-position cost of a full sd4 search in one live comparison,
+        # this was almost certainly the dominant cost of every
+        # eval_fen[s_batched]/qsearch_fen[s_batched] call this class has
+        # ever made, not anything about eval/qsearch being expensive.
+        self._send("nopost")
         atexit.register(self._hard_cleanup)
 
     def __enter__(self):
@@ -467,6 +502,8 @@ def filter_by_qsearch_agreement(engine, positions, max_diff_cp=30,
     load_training_positions' module comment: qsearch there would need the
     book disabled, a real time budget, and still return a useless 0 swing
     for tactics that resolve into recognized draws/endgames)."""
+    import time as _time
+    t0 = _time.time()
     kept = []
     n_dropped = 0
     for i in range(0, len(positions), batch_size):
@@ -482,12 +519,86 @@ def filter_by_qsearch_agreement(engine, positions, max_diff_cp=30,
                 kept.append((fen, result))
             else:
                 n_dropped += 1
+        done = i + len(chunk)
+        elapsed = _time.time() - t0
+        rate = done / elapsed if elapsed > 0 else 0
+        eta = (len(positions) - done) / rate if rate > 0 else float("inf")
+        print(f"  ...{done}/{len(positions)} checked "
+              f"({len(kept)} kept, {n_dropped} dropped) "
+              f"[{rate:.0f}/s, eta {eta/60:.1f}m]")
     print(f"  qsearch-agreement filter: kept {len(kept)}/{len(positions)} "
           f"(dropped {n_dropped} where |eval-qsearch| > {max_diff_cp}cp "
           f"or either was a mate score)")
     return kept
 
 
+def load_quiet_position_pool(engine, engine_path, pgn_path, max_positions=200_000,
+                              sample_every=8, min_elo=2400, game_stride=1,
+                              qsearch_max_diff=None, cache_dir="position_cache",
+                              force_rescan=False):
+    """Combines load_training_positions (SEE-quiet PGN scan, already its
+    own cache) with filter_by_qsearch_agreement (the stricter, engine-
+    driven gate) behind ONE cache keyed on everything that could change
+    which positions end up in the final pool -- so a repeat run with the
+    same settings against the same engine build can skip straight past
+    both the PGN scan AND the qsearch pass entirely, not just the scan.
+
+    The qsearch-agreement result depends on the actual engine binary and
+    whatever DNA is currently loaded into it (a different eval.c build,
+    or different baseline constants, can legitimately classify a
+    different subset of positions as "quiet"), unlike the pure-Python
+    SEE filter -- so this cache key includes the binary's mtime+size and
+    a hash of its currently-loaded baseline DNA on top of
+    load_training_positions' own key material, and is intentionally a
+    SEPARATE cache file rather than folded into that one, so a binary
+    rebuild invalidates only the qsearch-dependent layer, not the
+    (unaffected) raw PGN-scan result underneath it.
+
+    force_rescan forces both layers to recompute, matching
+    load_training_positions' own flag semantics.
+    """
+    raw_positions = load_training_positions(
+        pgn_path, max_positions=max_positions, sample_every=sample_every,
+        min_elo=min_elo, game_stride=game_stride, cache_dir=cache_dir,
+        force_rescan=force_rescan,
+    )
+    if qsearch_max_diff is None:
+        return raw_positions
+
+    engine_path = Path(engine_path)
+    est = engine_path.stat()
+    baseline_flat, _ = engine.dump_dna()
+    dna_hash = hashlib.sha256(str(baseline_flat).encode()).hexdigest()[:16]
+    key_material = (
+        f"{pgn_path}|{max_positions}|{sample_every}|{min_elo}|{game_stride}|"
+        f"qsearch|{qsearch_max_diff}|{engine_path}|{est.st_mtime}|"
+        f"{est.st_size}|{dna_hash}"
+    )
+    key = hashlib.sha256(key_material.encode()).hexdigest()[:16]
+    cache_path = Path(cache_dir) / f"quiet_pool_{key}.tsv"
+
+    if cache_path.exists() and not force_rescan:
+        positions = []
+        with open(cache_path) as f:
+            for line in f:
+                fen, result = line.rstrip("\n").split("\t")
+                positions.append((fen, float(result)))
+        print(f"  loaded {len(positions)} qsearch-filtered positions from "
+              f"cache {cache_path}")
+        return positions
+
+    print(f"Filtering by eval/qsearch agreement (max_diff={qsearch_max_diff}cp)...")
+    filtered = filter_by_qsearch_agreement(
+        engine, raw_positions, max_diff_cp=qsearch_max_diff
+    )
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(cache_path, "w") as f:
+        for fen, result in filtered:
+            f.write(f"{fen}\t{result}\n")
+    print(f"  cached {len(filtered)} qsearch-filtered positions to {cache_path}")
+    return filtered
+
+
 def load_training_positions(pgn_path, max_positions=200_000, sample_every=8,
                              min_elo=2400, game_stride=1,
                              cache_dir="position_cache", force_rescan=False):
@@ -592,6 +703,9 @@ def _load_training_positions_uncached(pgn_path, max_positions=200_000,
                 ply += 1
                 if is_quiet_training_ply(board, ply, was_capture, sample_every):
                     positions.append((board.fen(), r))
+                    if len(positions) % 20000 == 0:
+                        print(f"  ...{len(positions)} positions extracted "
+                              f"({game_index} games scanned so far)")
     return positions
 
 
@@ -890,24 +1004,33 @@ def write_dna_file(path, flat, row_lengths):
 def tune(engine_path, pgn_path, dna_scratch_dir, out_path,
          batch_size=None, holdout_frac=0.1, full_eval_every=1, max_passes=6,
          n_workers=2, max_positions=200_000, min_elo=2400, frozen_arrays=None,
-         game_stride=1, force_rescan=False, qsearch_max_diff=None):
+         game_stride=1, force_rescan=False, qsearch_max_diff=None,
+         position_file=None):
     """batch_size=None (the default) sizes each pass's training batch as
     1/10th of the training pool (post-holdout-split) instead of a fixed
     count -- a fixed 2000 was fine at the 25k-position scale this was
     first tuned against, but silently stayed thin (~4 positions per
     tunable parameter) once runs started loading 600k positions,
     since nothing scaled the batch with the pool. Pass an explicit int
-    to override."""
+    to override.
+
+    position_file (2026-09-15): a pre-built (fen, result) TSV -- e.g.
+    from build_curated_texel_pool.py's matched mate/control pool --
+    bypasses load_quiet_position_pool (PGN scan + qsearch filter)
+    entirely and loads this file directly instead. pgn_path is ignored
+    when set, but still required positionally for call-site stability.
+    """
     import time as _time
     run_start = _time.time()
 
     engine = EnginePool(engine_path, dna_scratch_dir, n_workers=n_workers)
     try:
-        _tune_body(engine, pgn_path, out_path, batch_size, holdout_frac,
-                   full_eval_every, max_passes, max_positions, min_elo,
-                   run_start, frozen_arrays=frozen_arrays,
+        _tune_body(engine, engine_path, pgn_path, out_path, batch_size,
+                   holdout_frac, full_eval_every, max_passes, max_positions,
+                   min_elo, run_start, frozen_arrays=frozen_arrays,
                    game_stride=game_stride, force_rescan=force_rescan,
-                   qsearch_max_diff=qsearch_max_diff)
+                   qsearch_max_diff=qsearch_max_diff,
+                   position_file=position_file)
     finally:
         # Graceful quit() is what actually releases each engine's
         # SysV semaphore (input.c) -- a bare kill() does not, since it
@@ -917,10 +1040,10 @@ def tune(engine_path, pgn_path, dna_scratch_dir, out_path,
         engine.quit()
 
 
-def _tune_body(engine, pgn_path, out_path, batch_size, holdout_frac,
+def _tune_body(engine, engine_path, pgn_path, out_path, batch_size, holdout_frac,
                full_eval_every, max_passes, max_positions, min_elo, run_start,
                frozen_arrays=None, game_stride=1, force_rescan=False,
-               qsearch_max_diff=None):
+               qsearch_max_diff=None, position_file=None):
     import time as _time
     baseline_flat, row_lengths = engine.dump_dna()
 
@@ -936,21 +1059,24 @@ def _tune_body(engine, pgn_path, out_path, batch_size, holdout_frac,
         print(f"Freezing {n_frozen}/{len(param_vec)} parameters in "
               f"{len(frozen_arrays)} array(s): {sorted(frozen_arrays)}")
 
-    print(f"Loading up to {max_positions} quiet positions (min_elo={min_elo}, "
-          f"game_stride={game_stride})...")
     t0 = _time.time()
-    all_positions = load_training_positions(
-        pgn_path, max_positions=max_positions, min_elo=min_elo,
-        game_stride=game_stride, force_rescan=force_rescan
-    )
-
-    if qsearch_max_diff is not None:
-        print(f"Filtering by eval/qsearch agreement (max_diff={qsearch_max_diff}cp)...")
-        t_qs = _time.time()
-        all_positions = filter_by_qsearch_agreement(
-            engine, all_positions, max_diff_cp=qsearch_max_diff
+    if position_file is not None:
+        print(f"Loading pre-built position pool from {position_file} "
+              f"(bypassing PGN scan + qsearch filter -- already done "
+              f"when this file was built)...")
+        all_positions = []
+        with open(position_file) as f:
+            for line in f:
+                fen, result = line.rstrip("\n").split("\t")
+                all_positions.append((fen, float(result)))
+    else:
+        print(f"Loading up to {max_positions} quiet positions (min_elo={min_elo}, "
+              f"game_stride={game_stride})...")
+        all_positions = load_quiet_position_pool(
+            engine, engine_path, pgn_path, max_positions=max_positions,
+            min_elo=min_elo, game_stride=game_stride,
+            qsearch_max_diff=qsearch_max_diff, force_rescan=force_rescan
         )
-        print(f"  filtered in {_time.time()-t_qs:.1f}s")
 
     random.shuffle(all_positions)
     print(f"  got {len(all_positions)} positions in {_time.time()-t0:.1f}s")
@@ -1168,6 +1294,16 @@ if __name__ == "__main__":
     if no_pgn_elo_filter:
         args.remove("--no-pgn-elo-filter")
 
+    # --position-file: a pre-built (fen, result) TSV (e.g. from
+    # build_curated_texel_pool.py) -- bypasses the PGN scan + qsearch
+    # filter entirely. pgn_path is still required positionally but
+    # ignored when this is set.
+    position_file = None
+    if "--position-file" in args:
+        idx = args.index("--position-file")
+        position_file = args[idx + 1]
+        del args[idx:idx + 2]
+
     frozen_arrays = set()
     for flag in ("--freeze", "--freeze="):
         for a in list(args):
@@ -1182,6 +1318,8 @@ if __name__ == "__main__":
                 break
 
     kwargs = {}
+    if position_file is not None:
+        kwargs["position_file"] = position_file
     if frozen_arrays:
         kwargs["frozen_arrays"] = frozen_arrays
     if game_stride != 1: