NNUE search integration: FH% tables, NODE_TYPE refinement, TREEDUMP rewrite

- data.c/chess.h: widen the joint FH% table's late axis 16->17, dedicating
  the new bucket 0 to a whole-node ("will something here fail high at
  all") sample rather than sharing it with the first-leftover-move curve.
  Add an NNUE-eval counterpart table (g_uJointFHThousandths_NNUE) and an
  NNUE counterpart of the beta-side table, selected at runtime based on
  whether NNUE eval is active.
- search.c/searchsup.c: add _RefineNodeTypeGuess, which uses the FH%
  tables (via g_NodeTypeGuessTable) to refine a node's NODE_TYPE guess
  instead of the old heuristic; thread uParentMoveFHEstimate through
  PLY_INFO so a node can see its parent's prediction for the move that
  led to it.
- generate.c: initialize sKillers[2]/[3].uBonus unconditionally (an
  uninitialized-stack read caught by valgrind under LTO; harmless today
  but a real bug).
- dumptree.c/chess.h/GNUmakefile: replace the long-dead DUMP_TREE XML
  tree dump (written for a GUI this project no longer has) with TREEDUMP,
  a flat, path-keyed move/result trace meant to diff against an
  identically-instrumented Stockfish build -- see docs/TREEDUMP_FORMAT.md.
- docs/DEEPER_PRUNING.md: notes on closing the search-efficiency gap vs
  Stockfish.
- crafty_egtb/stockfish_xboard/typhoon_nnue: wrapper scripts so
  match_play.py's Engine class can drive Crafty, Stockfish (via
  PolyGlot), and an NNUE-enabled typhoon the same way it drives HCE
  typhoon.
- tests/lop: rename lop.txt -> lop.ep_; update compare_lop.cpp.
- annotate_pgns: add aggregate_marginsafety.py, dedup_pgn.py,
  fit_lazy_probe.py; update annotate.py/run_nnue_pipeline.py for the NNUE
  training pipeline.
- .gitignore: ignore compile_commands.json (clangd build DB) and
  eval.nnue (a symlink into the untracked head_reference/ comparison
  baseline), neither of which belongs in the repo.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

1 files changed, 173 insertions, 243 deletions

diff --git a/src/dumptree.c b/src/dumptree.c
index 7a8f9e3..c9527bc 100644
--- a/src/dumptree.c
+++ b/src/dumptree.c
@@ -8,55 +8,47 @@ Module Name:
 
 Abstract:
 
-    Support routines for dumping a search tree into an XML file for
-    offline analysis by typhoonui.exe.
+    TREEDUMP: a flat, line-oriented, path-keyed node/move trace of the
+    search tree, meant to be diffed against an identically-instrumented
+    Stockfish build on the same FEN. See TREEDUMP_FORMAT.md for the
+    output format and chess.h's TREEDUMP comment block for the calling
+    convention.
 
-Author:
+    Build-time-only ("gmake TREEDUMP=1 ..."), always-on-when-built --
+    there is no runtime toggle/filter. Keep a dump small by using a
+    shallow "sd N"/"go depth N", not by restricting what gets written.
+
+    This replaces an earlier (2004), long-unused tree-dump mechanism
+    that wrote XML for a Windows GUI (typhoonui.exe) long since gone;
+    that code and its DUMP_TREE build flag have been removed outright
+    rather than kept alongside this one, on the assumption that a
+    mechanism nothing has exercised in two decades is broken.
 
-    Scott Gasch ([email protected]) 17 Jul 2004
+Author:
 
-Revision History:
+    Scott Gasch ([email protected]) 17 Jul 2004, replaced 2026-10-22
 
 **/
 
-#ifdef DUMP_TREE
+#ifdef TREEDUMP
 #include "chess.h"
 
-#define NHUMAN_READABLE
-#ifdef HUMAN_READABLE
-#define SPACES_TO_INDENT_PER_PLY (2)
-#else
-#define SPACES_TO_INDENT_PER_PLY (0)
-#endif
-#define DISK_SECTOR_SIZE_BYTES   (512)
-#define NUM_SECTORS_TO_BUFFER    (2)
+static FILE *g_pfTreeDump = NULL;
 
-FILE *pfTreeDump = NULL;
-ULONG g_uBufferSize = 0;
-ULONG g_uBufferUsed = 0;
-CHAR *g_Buffer = NULL;
-#ifdef MP
-volatile static ULONG g_uBufferLock = 0;
-#define BUF_IS_LOCKED (g_uBufferLock != 0)
-#define LOCK_BUF \
-    AcquireSpinLock(&g_uBufferLock); \
-    ASSERT(BUF_IS_LOCKED);
-#define UNLOCK_BUF \
-    ASSERT(BUF_IS_LOCKED); \
-    ReleaseSpinLock(&g_uBufferLock);
-#else // no MP
-#define BUF_IS_LOCKED
-#define LOCK_BUF
-#define UNLOCK_BUF
-#endif
-
-void 
-InitializeTreeDump(void)
+void
+TDOpen(void)
 /**
 
 Routine description:
 
-    Initialize tree dumping system.
+    Open (truncate) the fixed-name TREEDUMP output file. Call once at
+    startup when built with TREEDUMP -- there is no runtime toggle, so
+    this always runs and the file always gets every node/move of every
+    search performed. fflush() after every single line (unbuffered)
+    since TREEDUMP runs are expected to be small (a shallow sd/go
+    depth), not performance sensitive, and a flushed file is
+    immediately diffable even if the engine is killed mid-run or
+    crashes.
 
 Parameters:
 
@@ -68,62 +60,26 @@ Return value:
 
 **/
 {
-    //
-    // Allocate an internal buffer so that our writes to disk
-    // approximate a sector size.
-    //
-    g_uBufferSize = DISK_SECTOR_SIZE_BYTES * NUM_SECTORS_TO_BUFFER;
-    g_Buffer = SystemAllocateMemory(g_uBufferSize);
-
-    //
-    // Here's the file we'll want to write to...
-    //
-    pfTreeDump = fopen("typhoon.xml", "wb+");
-    if (NULL == pfTreeDump)
+    g_pfTreeDump = fopen("treedump.log", "w");
+    if (NULL == g_pfTreeDump)
     {
-        UtilPanic(FATAL_ACCESS_DENIED,
-                  NULL, "typhoon.xml", "write", "NULL", __FILE__, __LINE__);
+        UtilPanic(FATAL_ACCESS_DENIED, NULL, "treedump.log", "write",
+                  "NULL", __FILE__, __LINE__);
     }
-    fprintf(pfTreeDump, "<?xml version=\"1.0\" encoding=\"utf-8\""
-            " standalone=\"no\"?>\n");
 }
 
-static 
-void _FlushTheBuffer(void)
-/**
-
-Routine description:
-
-    Flush the buffer to the disk file.
-
-Parameters:
-
-    void
-
-Return value:
 
-    void
-
-**/
-{
-    fprintf(pfTreeDump, g_Buffer);
-    g_Buffer[0] = '\0';
-    g_uBufferUsed = 0;
-}
-
-
-static void CDECL 
-_AppendToBuffer(CHAR *buf)
+void
+TDClose(void)
 /**
 
 Routine description:
 
-    Append a string to the buffer; possibly flush the buffer to disk
-    first to make enough room for the new string.
+    Flush and close the TREEDUMP output file.
 
 Parameters:
 
-    CHAR *buf : the string to append
+    void
 
 Return value:
 
@@ -131,116 +87,111 @@ Return value:
 
 **/
 {
-    ULONG uSpaceNeeded;
-    ASSERT(BUF_IS_LOCKED);
-    
-    //
-    // How much space do we need?
-    //
-    uSpaceNeeded = strlen(buf);
-    ASSERT(uSpaceNeeded < MEDIUM_STRING_LEN_CHAR);
-    ASSERT(uSpaceNeeded < g_uBufferSize);
-    
-    //
-    // See if we need to flush the real buffer to the disk before we can
-    // append this new message.
-    // 
-    LOCK_BUF;
-    if (g_uBufferUsed + uSpaceNeeded >= g_uBufferSize)
+    if (NULL != g_pfTreeDump)
     {
-        _FlushTheBuffer();
-        ASSERT(g_uBufferUsed + uSpaceNeeded < g_uBufferSize);
+        fflush(g_pfTreeDump);
+        fclose(g_pfTreeDump);
+        g_pfTreeDump = NULL;
     }
-    
-    //
-    // Append the new data to the big buffer
-    //
-    strcat(g_Buffer, buf);
-    g_uBufferUsed += uSpaceNeeded;
-    ASSERT(g_uBufferUsed < g_uBufferSize);
-    UNLOCK_BUF;
 }
 
 
-static void _IndentTreeDumpMessage(ULONG uPly)
-/**
-
-Routine description:
-
-    Append a number of spaces to the buffer to indent the next output
-    appropriately based upon the current search ply.
-
-Parameters:
-
-    ULONG uPly : the current search ply
-
-Return value:
-
-    void
-
-**/
+// Formats a move as a UCI/ICS-style coordinate string ("e2e4"), or the
+// literal "0000" for a null move (MOVE{0}, same sentinel as NULLMOVE in
+// data.c) -- matches Stockfish's UCIEngine::move()'s spelling for
+// Move::null() exactly, so the two engines' logs tokenize identically.
+// Not thread safe (static buffer, same caveat as MoveToIcs itself,
+// which this wraps) -- TREEDUMP builds are meant for single-threaded
+// diagnostic runs only, matching this project's own comparison
+// methodology (see CLAUDE.md's "--cpus 1" convention).
+static CHAR *
+_TDMoveToStr(MOVE mv)
 {
-    ULONG uSpaces = uPly * SPACES_TO_INDENT_PER_PLY;
-    ASSERT(uSpaces < g_uBufferSizeBytes);
-    ASSERT(HUMAN_READABLE);
-    while(uSpaces)
+    static CHAR szNull[] = "0000";
+    CHAR *p;
+
+    if (0 == mv.uMove)
     {
-        _AppendToBuffer(" ");
-        uSpaces--;
+        return(szNull);
     }
+    p = MoveToIcs(mv);
+    ASSERT(p);
+    return(p);
 }
 
 
-void CDECL 
-DTTrace(ULONG uPly, CHAR *szMessage, ...)
-/**
-
-Routine description:
-
-    A printf-like routine for adding a message to the buffer / dumpfile.
-    Note: messages that exceed MEDIUM_STRING_LEN_CHAR are truncated.
-
-Parameters:
-
-    ULONG uPly : the current search ply
-    CHAR *szMessage : the message format string
-    ... : variable number of interpolated arguments
-
-Return value:
-
-    void 
-
-**/
+// Builds the path of moves from the root up to and including ctx's
+// current ply's own incoming move -- i.e. ctx->sPlyInfo[0..ctx->uPly-1].
+// MakeMove() (move.c) stores each move at the OLD ply index before
+// incrementing ctx->uPly, so this naturally includes:
+//   - the move just made by a caller logging from inside SearchOneMove
+//     (called after MakeMove, before UnmakeMove -- ctx->uPly is already
+//     bumped to the child ply, so sPlyInfo[childPly - 1] == that move), or
+//   - only the moves leading INTO this node, with none of this node's
+//     own not-yet-tried moves, for a caller logging a node's own RESULT
+//     from its own ply (e.g. search.c's "end:").
+// Same role as the Stockfish side's TreeDump::path() (search.cpp) --
+// see TREEDUMP_FORMAT.md. Not thread safe (static buffer); see
+// _TDMoveToStr's comment.
+static CHAR *
+_TDPath(SEARCHER_THREAD_CONTEXT *ctx)
 {
-    va_list ap;
-    CHAR buf[MEDIUM_STRING_LEN_CHAR];
-    buf[0] = '\0';
+    static CHAR szPath[2048];
+    ULONG p;
 
-    //
-    // Populate the local buffer
-    //
-    va_start(ap, szMessage);
-    COMPILER_VSNPRINTF(buf, MEDIUM_STRING_LEN_CHAR - 2, szMessage, ap);
-    va_end(ap);
-#ifdef HUMAN_READABLE
-    strcat(buf, "\n");
-    _IndentTreeDumpMessage(uPly);
-#endif
-    _AppendToBuffer(buf);
+    szPath[0] = '\0';
+    for (p = 0; p < ctx->uPly; p++)
+    {
+        if ('\0' != szPath[0])
+        {
+            strncat(szPath, " ", ARRAY_LENGTH(szPath) - strlen(szPath) - 1);
+        }
+        strncat(szPath, _TDMoveToStr(ctx->sPlyInfo[p].mv),
+                ARRAY_LENGTH(szPath) - strlen(szPath) - 1);
+    }
+    return(szPath);
 }
 
 
-void 
-CleanupTreeDump(void)
+void
+TDMove(SEARCHER_THREAD_CONTEXT *ctx,
+      ULONG uDepth,
+      SCORE iAlpha,
+      SCORE iBeta,
+      CHAR *szKind,
+      CHAR *szAct,
+      double dReducedPlies,
+      double dExtendedPlies,
+      SCORE iScore,
+      FLAG fSkipped)
 /**
 
 Routine description:
 
-    Tear down the tree dumping system
+    Log one MOVE line: a decision about the move already made at
+    ctx's current (child) ply -- see this file's header comment for
+    the calling convention (after MakeMove, before UnmakeMove).
 
 Parameters:
 
-    void
+    SEARCHER_THREAD_CONTEXT *ctx : current searcher thread context,
+                    already past MakeMove for the move being logged.
+    ULONG uDepth    : remaining depth (in internal ONE_PLY ticks) at
+                    the PARENT node, before this move's own extension/
+                    reduction.
+    SCORE iAlpha, iBeta : the parent node's alpha/beta.
+    CHAR *szKind    : move classification, e.g. "FIRST", "CAP", "QUIET",
+                    "NULL", freeform.
+    CHAR *szAct     : what happened to it, e.g. "FULL", "LMR", "LMP",
+                    "NMP", freeform.
+    double dReducedPlies, dExtendedPlies : magnitude, in whole plies,
+                    of any reduction/extension applied (0.0 if none).
+    SCORE iScore    : the score returned by searching this move;
+                    ignored (but still required as a parameter for a
+                    uniform call shape) when fSkipped is TRUE.
+    FLAG fSkipped   : TRUE if this move was pruned/skipped without
+                    being searched at all (e.g. LMP) -- logs
+                    "score=SKIPPED" instead of a numeric score.
 
 Return value:
 
@@ -248,44 +199,58 @@ Return value:
 
 **/
 {
-    //
-    // Cleanup file and buffer
-    //
-    if (NULL != pfTreeDump)
+    if (NULL == g_pfTreeDump)
     {
-        _FlushTheBuffer();
-        fflush(pfTreeDump);
-        fclose(pfTreeDump);
+        return;
     }
-
-    if (NULL != g_Buffer)
+    if (TRUE == fSkipped)
     {
-        SystemFreeMemory(g_Buffer);
-        g_Buffer = NULL;
-        g_uBufferSize = 0;
+        fprintf(g_pfTreeDump,
+                "MOVE path=\"%s\" depth=%.2f alpha=%d beta=%d kind=%s "
+                "act=%s r=%.2f ext=%.2f score=SKIPPED\n",
+                _TDPath(ctx), (double)uDepth / ONE_PLY, iAlpha, iBeta,
+                szKind, szAct, dReducedPlies, dExtendedPlies);
     }
+    else
+    {
+        fprintf(g_pfTreeDump,
+                "MOVE path=\"%s\" depth=%.2f alpha=%d beta=%d kind=%s "
+                "act=%s r=%.2f ext=%.2f score=%d\n",
+                _TDPath(ctx), (double)uDepth / ONE_PLY, iAlpha, iBeta,
+                szKind, szAct, dReducedPlies, dExtendedPlies, iScore);
+    }
+    fflush(g_pfTreeDump);
 }
 
 
-void 
-DTEnterNode(SEARCHER_THREAD_CONTEXT *ctx,
-            ULONG uDepth, 
-            FLAG fIsQNode,
-            SCORE iAlpha, 
-            SCORE iBeta)
+void
+TDResult(SEARCHER_THREAD_CONTEXT *ctx,
+        ULONG uDepth,
+        SCORE iAlpha,
+        SCORE iBeta,
+        SCORE iEval,
+        SCORE iBestScore)
 /**
 
 Routine description:
 
-    Called by search or qsearch upon entering a new node.
+    Log one RESULT line: this node's own final outcome. Call from the
+    node's own ply (i.e. before any further move has been made) --
+    see this file's header comment.
 
 Parameters:
 
-    SEARCHER_THREAD_CONTEXT *ctx : searcher context
-    ULONG uDepth : depth
-    FLAG fIsQNode : is it a q node?
-    SCORE iAlpha : current alpha
-    SCORE iBeta : current beta
+    SEARCHER_THREAD_CONTEXT *ctx : current searcher thread context, at
+                    the node whose result is being logged.
+    ULONG uDepth    : remaining depth (internal ONE_PLY ticks) this
+                    node was searched to.
+    SCORE iAlpha, iBeta : this node's ORIGINAL (pre-widening) alpha/beta
+                    -- callers should pass iInitialAlpha, not whatever
+                    iAlpha was mutated to internally, so the bound
+                    classification below means what it says.
+    SCORE iEval     : this node's static eval (0/stale if the node
+                    returned before Eval() ran -- diagnostic only).
+    SCORE iBestScore : this node's final score.
 
 Return value:
 
@@ -293,65 +258,30 @@ Return value:
 
 **/
 {
-    ULONG uPly = ctx->uPly;
-    UINT64 u64NodeNum = ctx->sCounters.tree.u64TotalNodeCount;
-    CHAR *p;
-    
-    if (FALSE == fIsQNode)
+    CHAR *szBound;
+
+    if (NULL == g_pfTreeDump)
     {
-        DTTrace(uPly, "<n n=\"%"
-                COMPILER_LONGLONG_UNSIGNED_FORMAT 
-                "\">", u64NodeNum);
+        return;
     }
-    else
+    if (iBestScore >= iBeta)
     {
-        DTTrace(uPly, "<qn n=\"%" 
-                COMPILER_LONGLONG_UNSIGNED_FORMAT 
-                "\">", u64NodeNum);
-        p = PositionToFen(&ctx->sPosition);
-        ASSERT(p);
-        DTTrace(uPly + 1, "<fen p=\"%s\"/>", p);
-        SystemFreeMemory(p);
-        DTTrace(uPly + 1, "<i ab=\"%d, %d\" ply=\"%u\" depth=\"%u\"/>",
-                iAlpha, iBeta, uPly, uDepth);
+        szBound = "LOWER";
     }
-}
-
-
-void 
-DTLeaveNode(SEARCHER_THREAD_CONTEXT *ctx, 
-            FLAG fQNode,
-            SCORE iBestScore,
-            MOVE mvBestMove)
-/**
-
-Routine description:
-
-    Called by the search upon leaving a normal (non q) node.
-
-Parameters:
-
-    SEARCHER_THREAD_CONTEXT *ctx : searcher context
-    SCORE iBestScore : best score found
-    MOVE mvBestMove : best move found
-
-Return value:
-
-    void
-
-**/
-{
-    ULONG uPly = ctx->uPly;
-    
-    DTTrace(uPly + 1, "<d score=\"%d\" mv=\"%s\"/>",
-            iBestScore, MoveToSan(mvBestMove, &ctx->sPosition));
-    if (!fQNode) 
+    else if (iBestScore <= iAlpha)
     {
-        DTTrace(uPly, "</n>");
+        szBound = "UPPER";
     }
     else
     {
-        DTTrace(uPly, "</qn>");
+        szBound = "EXACT";
     }
+    fprintf(g_pfTreeDump,
+            "RESULT path=\"%s\" depth=%.2f alpha=%d beta=%d eval=%d "
+            "bound=%s score=%d\n",
+            _TDPath(ctx), (double)uDepth / ONE_PLY, iAlpha, iBeta, iEval,
+            szBound, iBestScore);
+    fflush(g_pfTreeDump);
 }
-#endif
+
+#endif // TREEDUMP