Phase A NNUE: feature-indexing spec, annotation/training pipeline, dump nnuefeatures

Builds out the NNUE Phase A prototype per docs/NNUE.md:

- src/nnuefeatures.c: standalone HalfKAv2_hm feature-index reference
  implementation (new `dump nnuefeatures` command), sourced from
  Stockfish's half_ka_v2_hm.{h,cpp} (sf_16 tag), including the 0x88->64
  square conversion typhoon's own board representation needs.
- annotate_pgns/: sharded TWIC/self-play annotation harness
  (annotate.py), position extraction, quiet-position filtering (flat
  and game-shaped, plus an upstream --prefilter-quiet option),
  train/holdout game-level stratified split, WDL-sigmoid K fitting,
  the Python HalfKAv2_hm reference encoder (nnue_feature_spec.py) and
  its bit-for-bit cross-checker against the C side
  (cross_check_features.py), and a first PyTorch training pipeline
  (nnue_model.py, nnue_train.py, build_training_tensors.py) -- a
  256x2-32-32-1-ish net trained locally on CPU against ~1.85M quiet
  TWIC positions as a pipeline/framework validation pass.
- Finish an in-progress InitMagic/InitLMRTable/InitEval ->
  InitializeBitboardMagic/InitializeLMRTable/InitializeEval rename:
  chess.h's prototypes hadn't been updated to match the already-renamed
  definitions/call sites, leaving 4 implicit-function-declaration
  warnings.
- tests/arasan.epd: new curated EPD suite.
- docs/: add the project's design docs (previously untracked).

Co-Authored-By: Claude Sonnet 5 <[email protected]>

1 files changed, 160 insertions, 0 deletions

diff --git a/annotate_pgns/check_see_against_engine.py b/annotate_pgns/check_see_against_engine.py
new file mode 100644
index 0000000..4f6aab3
--- /dev/null
+++ b/annotate_pgns/check_see_against_engine.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Cross-check eval_tune/tune_eval_dna.py's Python static_exchange_eval()
+(a classic swap-list SEE that admittedly ignores pins/x-rays/
+discoveries -- see its docstring) against the engine's own real SEE(),
+on real positions from an annotate.py run.
+
+This has never been done before: static_exchange_eval() has been used
+by the eval-tuning pipeline's quiet-position filter since that
+existed, but nothing has ever actually verified it agrees with the
+engine it's trying to approximate. Worth doing once, as reusable
+tooling, since this pipeline (annotate -> quiet-filter -> train) is
+going to be re-run many times as the project continues.
+
+Mechanism: the engine already has a bulk oracle for exactly this,
+`seescores <filename>` (command.c) -- takes a file of "setboard <fen>"
+lines (the same format shard_NN.script already uses) and prints
+"SEESCORES <fen> | <san> | <see-value>" for every legal capture in
+every position in the file, no search involved. Sample N FENs from an
+existing positions.jsonl, write them to a scratch "setboard"-lines
+file, run `seescores` on it, and diff every (fen, move) pair's engine
+SEE value against the Python one.
+
+Usage:
+    python3 check_see_against_engine.py --positions runs/sd10_twic_2M/positions.jsonl \
+        --sample 2000
+"""
+import argparse
+import json
+import os
+import random
+import re
+import subprocess
+import sys
+import tempfile
+from collections import Counter
+from pathlib import Path
+
+_EVAL_TUNE_DIR = Path(__file__).resolve().parent.parent / "src" / "eval_tune"
+sys.path.insert(0, str(_EVAL_TUNE_DIR))
+import chess
+
+# See quiet_filter.py's comment: dna_diff (imported by tune_eval_dna)
+# reads "../eval.c" relative to CWD at import time, not to its own file.
+_cwd = os.getcwd()
+os.chdir(_EVAL_TUNE_DIR)
+try:
+    from tune_eval_dna import static_exchange_eval
+finally:
+    os.chdir(_cwd)
+
+SEESCORES_RE = re.compile(r"^SEESCORES (\S+(?: \S+)*?) \| (\S+) \| (-?\d+)$")
+
+
+def sample_fens(positions_path, n, seed):
+    fens = []
+    with open(positions_path) as f:
+        for line in f:
+            rec = json.loads(line)
+            if rec.get("exact"):
+                continue
+            fens.append(rec["fen"])
+    random.Random(seed).shuffle(fens)
+    return fens[:n]
+
+
+def run_seescores(engine_path, fens, scratch_dir):
+    script_path = Path(scratch_dir) / "see_check.script"
+    out_path = Path(scratch_dir) / "see_check.out"
+    with open(script_path, "w") as f:
+        for fen in fens:
+            f.write(f"setboard {fen}\n")
+
+    cmd = [str(engine_path), "--batch", "--command", f"seescores {script_path}"]
+    with open(out_path, "w") as out:
+        subprocess.run(cmd, stdout=out, stderr=subprocess.STDOUT, check=True)
+    return out_path
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--positions", required=True,
+                     help="positions.jsonl to sample FENs from")
+    ap.add_argument("--sample", type=int, default=2000)
+    ap.add_argument("--seed", type=int, default=0)
+    ap.add_argument("--engine", default="../src/typhoon")
+    args = ap.parse_args()
+
+    engine_path = Path(args.engine).resolve()
+    if not engine_path.exists():
+        sys.exit(f"engine not found: {engine_path}")
+
+    print(f"Sampling {args.sample} non-exact positions from {args.positions} ...")
+    fens = sample_fens(args.positions, args.sample, args.seed)
+    print(f"  got {len(fens)} FENs")
+
+    with tempfile.TemporaryDirectory() as scratch:
+        print("Running engine seescores ...")
+        out_path = run_seescores(engine_path, fens, scratch)
+
+        n_compared = 0
+        n_exact_match = 0
+        n_sign_match = 0
+        n_sign_mismatch = 0
+        worst = []  # (abs diff, fen, san, engine_val, python_val)
+        diff_histogram = Counter()
+
+        board_cache = {}
+        with open(out_path) as f:
+            for line in f:
+                m = SEESCORES_RE.match(line.rstrip("\n"))
+                if not m:
+                    continue
+                fen, san, engine_val = m.groups()
+                engine_val = int(engine_val)
+
+                board = board_cache.get(fen)
+                if board is None:
+                    board = chess.Board(fen)
+                    board_cache[fen] = board
+                try:
+                    move = board.parse_san(san)
+                except ValueError:
+                    continue
+                python_val = static_exchange_eval(board, move)
+
+                n_compared += 1
+                diff = engine_val - python_val
+                if diff == 0:
+                    n_exact_match += 1
+                bucket = "0" if diff == 0 else (
+                    "<=50" if abs(diff) <= 50 else
+                    "<=200" if abs(diff) <= 200 else ">200")
+                diff_histogram[bucket] += 1
+
+                engine_sign = (engine_val > 0) - (engine_val < 0)
+                python_sign = (python_val > 0) - (python_val < 0)
+                if engine_sign == python_sign:
+                    n_sign_match += 1
+                else:
+                    n_sign_mismatch += 1
+
+                worst.append((abs(diff), fen, san, engine_val, python_val))
+
+    worst.sort(reverse=True)
+
+    print(f"\nCompared {n_compared} (fen, capture) pairs")
+    print(f"  exact match:  {n_exact_match} ({n_exact_match/max(n_compared,1):.1%})")
+    print(f"  sign match:   {n_sign_match} ({n_sign_match/max(n_compared,1):.1%})")
+    print(f"  sign MISmatch:{n_sign_mismatch} ({n_sign_mismatch/max(n_compared,1):.1%})")
+    print(f"  |diff| histogram: {dict(diff_histogram)}")
+
+    if worst:
+        print(f"\nWorst {min(10, len(worst))} disagreements "
+              f"(|engine - python|, fen, san, engine_see, python_see):")
+        for d, fen, san, ev, pv in worst[:10]:
+            print(f"  {d:5d}  {fen}  {san}  engine={ev}  python={pv}")
+
+
+if __name__ == "__main__":
+    main()