path: root/src/fhcollect.c

/*

    fhcollect.c

    Per-thread binary record emission for the learned FH% regression
    collection (FH_REGRESSION_COLLECTION) -- see docs/REGRESSION.md.
    Replaces the old FH_TABLE_COLLECTION histogram design: each logged
    event (a whole-node row, or one row per move/null-move-probe tried
    at an eligible node) is written directly to a per-thread binary
    file, buffered in FH_COLLECT_BUFFER_RECORDS-sized batches to avoid
    a syscall per record. No cross-thread state and no merge-back --
    each thread's file is independent; the offline Python training
    pipeline concatenates/loads them together, not this process.

*/
#include "chess.h"

#ifdef FH_REGRESSION_COLLECTION

// Registry of every writer that's ever been opened, purely so
// FhCollectCloseAll (called once from MainProgramCleanup at real
// process shutdown) can find and flush every thread's file without
// main.c needing to hold a pointer to any SEARCHER_THREAD_CONTEXT --
// the root thread's own ctx is a function-static local inside root.c's
// Think()/Ponder(), not reachable from main.c directly.
#define FH_COLLECT_MAX_WRITERS (64)
static FH_COLLECT_WRITER *g_rgpFhCollectWriters[FH_COLLECT_MAX_WRITERS];
static ULONG g_uFhCollectWriterCount = 0;


void
FhCollectOpen(SEARCHER_THREAD_CONTEXT *ctx, ULONG uThreadNumber)
/**

Routine description:

    Open this thread's collection file if it isn't already open, and
    register its writer so FhCollectCloseAll can find it at shutdown.
    Safe to call more than once for the same ctx -- a no-op if the
    file is already open. This matters because ctx is frequently a
    reused, repeatedly-reinitialized struct (root.c's Think/Ponder use
    a function-static SEARCHER_THREAD_CONTEXT across many searches in
    one process) -- InitializeSearcherContext preserves an already-open
    pFile across its own memset specifically so this function only
    ever does real work once per thread's lifetime.

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx,
    ULONG uThreadNumber -- names the file; the root/main thread is 0,
        helper threads are split.c's uMyId + 1

Return value:

    void

**/
{
    FH_COLLECT_WRITER *w = &ctx->sFhCollectWriter;
    CHAR szPath[SMALL_STRING_LEN_CHAR];

    if (NULL != w->pFile)
    {
        return;
    }
    sprintf(szPath, "fhcollect_thread%02lu.bin", (unsigned long)uThreadNumber);
    w->pFile = fopen(szPath, "ab");
    if (NULL == w->pFile)
    {
        Bug("FhCollectOpen: unable to open '%s' for append.\n", szPath);
        return;
    }
    w->uBufferedCount = 0;
    if (g_uFhCollectWriterCount < FH_COLLECT_MAX_WRITERS)
    {
        g_rgpFhCollectWriters[g_uFhCollectWriterCount++] = w;
    }
    else
    {
        // Not fatal -- this writer just won't be proactively flushed
        // by FhCollectCloseAll; its buffered tail (up to
        // FH_COLLECT_BUFFER_RECORDS - 1 records) would only be lost if
        // the process is killed rather than exiting cleanly through
        // MainProgramCleanup. FH_COLLECT_MAX_WRITERS (64) comfortably
        // exceeds any realistic MAX_SPLITS + 1, so this should never
        // actually fire.
        Bug("FhCollectOpen: FH_COLLECT_MAX_WRITERS exceeded -- this "
            "writer won't be flushed by FhCollectCloseAll.\n");
    }
}


void
FhCollectFlush(SEARCHER_THREAD_CONTEXT *ctx)
/**

Routine description:

    Write out this thread's currently-buffered records, if any, and
    reset the buffer. Does not close the file.

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx

Return value:

    void

**/
{
    FH_COLLECT_WRITER *w = &ctx->sFhCollectWriter;

    if ((NULL != w->pFile) && (w->uBufferedCount > 0))
    {
        fwrite(w->rgBuffer, sizeof(FH_COLLECT_RECORD), w->uBufferedCount, w->pFile);
        w->uBufferedCount = 0;
    }
}


void
FhCollectEmit(SEARCHER_THREAD_CONTEXT *ctx, const FH_COLLECT_RECORD *prec)
/**

Routine description:

    Buffer one record for this thread, flushing the whole batch to
    disk once FH_COLLECT_BUFFER_RECORDS have accumulated. Opens the
    file lazily on first use if it isn't already open.

    Deliberately NOT opened eagerly from InitializeSearcherContext: a
    helper thread's ctx->uThreadNumber isn't set to its real value
    until immediately AFTER that call returns (split.c calls
    InitializeSearcherContext(NULL, ctx) then sets
    ctx->uThreadNumber = uMyId + 1 on the very next line) -- opening
    eagerly inside InitializeSearcherContext would race and every
    helper thread's first-ever open would collide on thread number 0.
    By the time any real search activity reaches this function,
    uThreadNumber is already correct for both the root thread (stays 0
    throughout) and every helper thread.

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx,
    const FH_COLLECT_RECORD *prec

Return value:

    void

**/
{
    FH_COLLECT_WRITER *w = &ctx->sFhCollectWriter;

    // Subsampling: keep roughly 1-in-N events, chosen independently
    // per event via randomMT() (mersenne.c) -- cheap, unbiased, and
    // preserves the exact joint continuous feature vector of every
    // KEPT row (unlike aggregating/bucketizing, which would
    // reintroduce the combinatorial cardinality problem this whole
    // design moved off the old table to avoid -- see
    // docs/REGRESSION.md's "On subsampling vs. aggregation" section).
    // Both denominators read once per process, not per call; unset or
    // invalid means no subsampling (denom 1, keep everything).
    //
    // TIER-DEPENDENT, not uniform (2026-10-24 session finding): the
    // beta-side per-move population (a real move/null-move/hash-move
    // tried after a QUICK_NULL_MOVE/static-cutoff check already said
    // this node looks won) is structurally RARE -- null-move or one
    // of the first moves resolves the cutoff the overwhelming
    // majority of the time, so actually reaching deeper into the move
    // list at an already-winning node is scarce almost by
    // construction, not something that gets proportionally more
    // common the longer collection runs. It's ALSO the one population
    // with no historical analog at all to fall back on (the old
    // table's beta side was whole-node-only, never per-move -- see
    // docs/REGRESSION.md). Applying the same aggressive denominator
    // there as to the overwhelmingly common alpha-side leftover
    // population was actively discarding an already-scarce, already-
    // irreplaceable signal for no real disk benefit (this rare
    // population's total volume is small regardless of its sampling
    // rate) -- so it gets its own, much gentler (default: none)
    // denominator instead. NO_MOVE/NULL/HASH on the beta side are NOT
    // rare this way (they fire at essentially every eligible node,
    // same as the alpha side) and stay on the common rate.
    static ULONG uSampleDenom = 0;
    static ULONG uRareSampleDenom = 0;
    if (0 == uSampleDenom)
    {
        CHAR *pszDenom = getenv("FH_COLLECT_SAMPLE_DENOM");
        uSampleDenom = (pszDenom && atol(pszDenom) > 0) ?
            (ULONG)atol(pszDenom) : 1;
    }
    if (0 == uRareSampleDenom)
    {
        CHAR *pszRareDenom = getenv("FH_COLLECT_SAMPLE_DENOM_RARE");
        uRareSampleDenom = (pszRareDenom && atol(pszRareDenom) > 0) ?
            (ULONG)atol(pszRareDenom) : 1;
    }
    {
        // "Common" = has real old-table backing AND dominates raw
        // volume; everything else is "rare" and gets the gentle
        // denominator instead, on EITHER table (2026-10-24 session
        // finding, generalized from beta-only after Scott's own
        // question -- alpha's HASH/GOOD_CAPTURE/KILLER/COUNTERMOVE/
        // NULL are exactly as old-table-absent as beta's, since
        // _ScoreAllMoves structurally excludes good captures and any
        // killer/countermove-bonus-bearing move from the
        // fIsLeftoverMove population the old alpha table actually
        // measured -- LEFTOVER_QUIET and LOSING_CAPTURE both stay
        // below GOOD_MOVE and so were both really covered historically,
        // but nothing else was, on either side).
        FLAG fIsCommon;
        if (FHC_TABLE_ALPHA == prec->bTable)
        {
            fIsCommon = (FHC_TIER_NO_MOVE == prec->bTier) ||
                        (FHC_TIER_LEFTOVER_QUIET == prec->bTier) ||
                        (FHC_TIER_LOSING_CAPTURE == prec->bTier);
        }
        else
        {
            fIsCommon = (FHC_TIER_NO_MOVE == prec->bTier);
        }
        // Depth-dependent, on top of tier (2026-10-24 follow-up
        // finding): a large REMAINING depth means this node sits
        // close to the root, where the branching factor makes nodes
        // structurally scarce compared to the shallow-remaining-depth
        // population near the leaves. Applying the common tier's
        // aggressive denominator uniformly across depth discards this
        // already-scarce deep tail at the SAME rate as the
        // overwhelmingly abundant shallow population -- so letting
        // collection run longer scales both up together and never
        // improves the deep tail's SHARE of the data. This is exactly
        // what the beta-side rigorous spot-check kept showing even
        // after hours of additional collection (depth-bucketed error
        // climbing from ~2pp at shallow depths to 37pp at the deepest,
        // with matched-cell sample counts (n_new) in the single/low
        // double digits there throughout). Fix: once remaining depth
        // passes a few plies, fall back to the gentle/rare
        // denominator regardless of tier -- cheap, since this
        // population is naturally small on its own.
        if (fIsCommon && ((prec->uRemainingDepthTicks / ONE_PLY) >= 3))
        {
            fIsCommon = FALSE;
        }
        ULONG uEffectiveDenom = fIsCommon ? uSampleDenom : uRareSampleDenom;
        if ((uEffectiveDenom > 1) && ((randomMT() % uEffectiveDenom) != 0))
        {
            return;
        }
    }

    if (NULL == w->pFile)
    {
        FhCollectOpen(ctx, ctx->uThreadNumber);
        if (NULL == w->pFile)
        {
            return; // open failed, already Bug()'d inside FhCollectOpen
        }
    }
    w->rgBuffer[w->uBufferedCount++] = *prec;
    if (w->uBufferedCount >= FH_COLLECT_BUFFER_RECORDS)
    {
        FhCollectFlush(ctx);
    }
}


void
FhCollectClose(SEARCHER_THREAD_CONTEXT *ctx)
/**

Routine description:

    Flush and close this one thread's file, and deregister it. Most
    callers want FhCollectCloseAll (used by MainProgramCleanup)
    instead; this per-ctx version exists for any future caller that
    tracks its own ctx directly and wants to close it before process
    shutdown.

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx

Return value:

    void

**/
{
    FH_COLLECT_WRITER *w = &ctx->sFhCollectWriter;
    ULONG u;

    if (NULL == w->pFile)
    {
        return;
    }
    FhCollectFlush(ctx);
    fclose(w->pFile);
    w->pFile = NULL;
    for (u = 0; u < g_uFhCollectWriterCount; u++)
    {
        if (g_rgpFhCollectWriters[u] == w)
        {
            g_rgpFhCollectWriters[u] = g_rgpFhCollectWriters[--g_uFhCollectWriterCount];
            break;
        }
    }
}


void
FhCollectCloseAll(void)
/**

Routine description:

    Flush and close every registered writer -- call once, from
    MainProgramCleanup, at real process shutdown, so no thread's final
    buffered-but-unflushed records (up to FH_COLLECT_BUFFER_RECORDS - 1
    of them, per thread) are silently dropped.

Parameters:

    void

Return value:

    void

**/
{
    ULONG u;

    for (u = 0; u < g_uFhCollectWriterCount; u++)
    {
        FH_COLLECT_WRITER *w = g_rgpFhCollectWriters[u];
        if (NULL != w->pFile)
        {
            if (w->uBufferedCount > 0)
            {
                fwrite(w->rgBuffer, sizeof(FH_COLLECT_RECORD), w->uBufferedCount, w->pFile);
                w->uBufferedCount = 0;
            }
            fclose(w->pFile);
            w->pFile = NULL;
        }
    }
    g_uFhCollectWriterCount = 0;
}


void
FhCollectNodeEntry(SEARCHER_THREAD_CONTEXT *ctx,
                   FH_COLLECT_PENDING *pp,
                   SCORE iEval, SCORE iAlpha, SCORE iBeta,
                   ULONG uDepthTicks,
                   SCORE iImprove, SCORE iEnemyOrOwnImprove,
                   NODE_TYPE eNodeTypeGuess,
                   UINT64 u64PositionSig,
                   ULONG uTotalNonPawnMaterial)
/**

Routine description:

    Decide this node's collection eligibility (iEval <= iAlpha -> alpha
    table, iEval >= iBeta -> beta table, otherwise ineligible -- a real
    PV node, out of scope for this collection per docs/REGRESSION.md)
    and stash everything FhCollectMoveTried/FhCollectNodeExit will need
    later into *pp. Does NOT emit anything yet -- the whole-node
    (FHC_TIER_NO_MOVE) row's label isn't known until the real move
    loop finishes, so its actual FhCollectEmit call happens in
    FhCollectNodeExit, not here.

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx,
    FH_COLLECT_PENDING *pp,
    SCORE iEval, SCORE iAlpha, SCORE iBeta,
    ULONG uDepthTicks,
    SCORE iImprove, SCORE iEnemyOrOwnImprove -- the beta table's own
        second improve axis (enemy-improving); unused/ignored for an
        alpha-side node, which has only one improve axis
    NODE_TYPE eNodeTypeGuess -- the caller's PARENT-derived guess, not
        the refined one (caller's responsibility, see chess.h's
        FH_COLLECT_PENDING comment)
    UINT64 u64PositionSig,
    ULONG uTotalNonPawnMaterial

Return value:

    void

**/
{
    (void)(ctx);

    memset(pp, 0, sizeof(*pp));
    if (iEval <= iAlpha)
    {
        pp->fEligible = TRUE;
        pp->bTable = FHC_TABLE_ALPHA;
        pp->iGap = (SCORE)(iAlpha - iEval);
    }
    else if (iEval >= iBeta)
    {
        pp->fEligible = TRUE;
        pp->bTable = FHC_TABLE_BETA;
        pp->iGap = (SCORE)(iEval - iBeta);
    }
    else
    {
        // A real PV node (alpha < eval < beta) -- out of scope for
        // this collection, see docs/REGRESSION.md's "Scope" section.
        pp->fEligible = FALSE;
        return;
    }
    pp->bNodeTypeGuess = (UCHAR)eNodeTypeGuess;
    pp->iImprove = iImprove;
    pp->iEnemyOrOwnImprove = iEnemyOrOwnImprove;
    pp->uRemainingDepthTicks = uDepthTicks;
    pp->u64PositionSig = u64PositionSig;
    pp->uGoodCapturesTriedNoCutoff = 0;
    pp->uTotalNonPawnMaterial = uTotalNonPawnMaterial;
}


void
FhCollectMoveTried(SEARCHER_THREAD_CONTEXT *ctx,
                   FH_COLLECT_PENDING *pp,
                   FH_COLLECT_TIER eTier,
                   SCORE iScore, SCORE iBeta,
                   ULONG uMoveOrdinal, SCORE iMoveValue,
                   ULONG uMoveFhPctThousandths, ULONG uMoveFhAttempts,
                   FLAG fMovesKnown, ULONG uTotalLegalMoves)
/**

Routine description:

    Emit one record for a single move (or the null-move probe, or the
    hash move) just tried at an eligible node, now that its own score
    is known -- unlike the whole-node row, an individual move's label
    (did IT resolve >= iBeta) is available immediately, no deferred
    commit needed.

    uGoodCapturesTriedNoCutoff is read into the emitted record BEFORE
    being updated for this move, so the value a reader sees on any one
    record reflects captures tried strictly before it, not including
    it -- matching what would actually be knowable while this move was
    being decided, were this feeding a live reduction/pruning decision
    instead of just being logged.

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx,
    FH_COLLECT_PENDING *pp,
    FH_COLLECT_TIER eTier,
    SCORE iScore, SCORE iBeta,
    ULONG uMoveOrdinal, SCORE iMoveValue,
    ULONG uMoveFhPctThousandths, ULONG uMoveFhAttempts,
    FLAG fMovesKnown, ULONG uTotalLegalMoves -- FALSE/0 for a move tried
        before real move generation (the null-move probe, the hash
        move -- see docs/REGRESSION.md's moves_known/imputation design)

Return value:

    void

**/
{
    FH_COLLECT_RECORD rec;

    if (FALSE == pp->fEligible)
    {
        return;
    }

    memset(&rec, 0, sizeof(rec));
    rec.u64PositionSig = pp->u64PositionSig;
    rec.iGap = pp->iGap;
    rec.iMoveValue = iMoveValue;
    rec.iImprove = pp->iImprove;
    rec.iEnemyOrOwnImprove = pp->iEnemyOrOwnImprove;
    rec.uRemainingDepthTicks = pp->uRemainingDepthTicks;
    rec.uMoveOrdinal = uMoveOrdinal;
    rec.uTotalLegalMoves = (TRUE == fMovesKnown) ? uTotalLegalMoves : 0;
    rec.uMoveFhPctThousandths = uMoveFhPctThousandths;
    rec.uMoveFhAttempts = uMoveFhAttempts;
    rec.uGoodCapturesTriedNoCutoff = pp->uGoodCapturesTriedNoCutoff;
    rec.uTotalNonPawnMaterial = pp->uTotalNonPawnMaterial;
    rec.bTable = pp->bTable;
    rec.bTier = (UCHAR)eTier;
    rec.bNodeTypeGuess = pp->bNodeTypeGuess;
    rec.bMovesKnown = (TRUE == fMovesKnown) ? 1 : 0;
    rec.bLabel = (iScore >= iBeta) ? 1 : 0;
    FhCollectEmit(ctx, &rec);

    if ((FHC_TIER_GOOD_CAPTURE == eTier) && (iScore < iBeta))
    {
        pp->uGoodCapturesTriedNoCutoff++;
    }
}


void
FhCollectNodeExit(SEARCHER_THREAD_CONTEXT *ctx,
                  FH_COLLECT_PENDING *pp,
                  SCORE iBestScore, SCORE iBeta, FLAG fAborted)
/**

Routine description:

    Emit the whole-node (FHC_TIER_NO_MOVE) row stashed by
    FhCollectNodeEntry, now that iBestScore's real fate against iBeta
    is known. Skipped entirely on an aborted/incomplete search
    (iBestScore may not reflect a fully-searched node in that case, and
    logging it as ground truth would bias the model toward whatever
    got searched before the clock/node-limit fired) or if the node was
    never eligible to begin with.

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx,
    FH_COLLECT_PENDING *pp,
    SCORE iBestScore, SCORE iBeta, FLAG fAborted

Return value:

    void

**/
{
    FH_COLLECT_RECORD rec;

    if ((FALSE == pp->fEligible) || (TRUE == fAborted) ||
        (!IS_VALID_SCORE(iBestScore)))
    {
        return;
    }

    memset(&rec, 0, sizeof(rec));
    rec.u64PositionSig = pp->u64PositionSig;
    rec.iGap = pp->iGap;
    rec.iMoveValue = 0;
    rec.iImprove = pp->iImprove;
    rec.iEnemyOrOwnImprove = pp->iEnemyOrOwnImprove;
    rec.uRemainingDepthTicks = pp->uRemainingDepthTicks;
    rec.uMoveOrdinal = 0;
    // Total legal moves is genuinely unknown to production at the
    // point this whole-node prediction would actually run (before the
    // move loop) -- logged as unknown here too, even though this
    // collection build happens to know it by now (the real loop has
    // already run by end:), so training data matches what's really
    // available at inference time. See docs/REGRESSION.md.
    rec.uTotalLegalMoves = 0;
    rec.uMoveFhPctThousandths = 0;
    rec.uGoodCapturesTriedNoCutoff = pp->uGoodCapturesTriedNoCutoff;
    rec.uTotalNonPawnMaterial = pp->uTotalNonPawnMaterial;
    rec.bTable = pp->bTable;
    rec.bTier = FHC_TIER_NO_MOVE;
    rec.bNodeTypeGuess = pp->bNodeTypeGuess;
    rec.bMovesKnown = FALSE;
    rec.bLabel = (iBestScore >= iBeta) ? 1 : 0;
    FhCollectEmit(ctx, &rec);
}


#endif // FH_REGRESSION_COLLECTION
// FhClassifyMoveTier itself moved to searchsup.c (always compiled,
// not gated by FH_REGRESSION_COLLECTION) -- fhmodel.c's production
// inference needs it unconditionally too, and it's pure/stateless (no
// dependency on anything collection-specific), so there's no reason
// to duplicate it or leave it gated. See searchsup.c for the
// implementation and chess.h for the now-unconditional declaration.