path: root/src/probe.c

/**

Copyright (c) Scott Gasch

Module Name:

    probe.c

Abstract:

    Routines for probing endgame tablebases.  This uses Ronald de
    Man's Syzygy tablebase format via Jon Dart's "Fathom" probing
    library (see fathom/) instead of Eugene Nalimov's original
    tablebases.  Syzygy tables store WDL (win/draw/loss) + DTZ
    (distance to zeroing move) rather than Nalimov's DTM (distance
    to mate); they are dramatically smaller on disk and the probing
    code is far simpler as a result.  We lose literal mate-distance
    information from interior-node probes, so a probed win/loss is
    reported as a large-but-not-mate-range score rather than a
    precise mate score.

Author:

    Scott Gasch ([email protected]) 02 Jul 2004
    Converted to Syzygy/Fathom 23 Aug 2026

Revision History:

    $Id: probe.c 355 2008-07-01 15:46:43Z scott $

**/

#include "chess.h"
#include "fathom/tbprobe.h"

// See eval.c's identical comment: routes this file's Syzygy-bitboard-
// builder bitboard-scan loops to the inlined
// FastCoorFromBitBoardRank8ToRank1 (chess.h) instead of the real
// out-of-line extern call.
#define CoorFromBitBoardRank8ToRank1 FastCoorFromBitBoardRank8ToRank1

ULONG g_uEgtbLock = 0;

//
// A probed tablebase win/loss doesn't carry a real distance-to-mate
// (Syzygy WDL doesn't encode one), so we report a large decisive
// score that stays safely below NMATE.  This keeps it from being
// mistaken for (and ply-adjusted as) a literal forced mate elsewhere
// in the engine while still dominating ordinary evaluation scores.
//
#define EGTB_WIN_SCORE              (+NMATE)
#define EGTB_LOSS_SCORE             (-NMATE)

static FLAG g_fEgtbInitialized = FALSE;

void
InitializeEGTB(void)
/**

Routine description:

    [Re]Initialize the Syzygy EGTB system.  Called during system
    startup and when the user uses "set" to change the EGTB path.

Parameters:

    void (uses g_Options.szEGTBPath)

Return value:

    void

**/
{
    CHAR *szPath = g_Options.szEGTBPath;

    if (TRUE == g_fEgtbInitialized)
    {
        tb_free();
        g_fEgtbInitialized = FALSE;
    }

    if ((szPath != NULL) && strlen(szPath) > 0)
    {
        if (TRUE == tb_init(szPath))
        {
            g_fEgtbInitialized = TRUE;
            if (TB_LARGEST > 0)
            {
                Trace("Found Syzygy endgame tablebases (up to %u men).\n\n",
                      TB_LARGEST);
            }
        }
    }
}

void
CleanupEGTB(void)
/**

Routine description:

    Cleanup the Syzygy EGTB system.

Parameters:

    void

Return value:

    void

**/
{
    if (TRUE == g_fEgtbInitialized)
    {
        tb_free();
        g_fEgtbInitialized = FALSE;
    }
}


static void
_BuildFathomBitboards(IN POSITION *pos,
                      OUT UINT64 *pu64White,
                      OUT UINT64 *pu64Black,
                      OUT UINT64 *pu64Kings,
                      OUT UINT64 *pu64Queens,
                      OUT UINT64 *pu64Rooks,
                      OUT UINT64 *pu64Bishops,
                      OUT UINT64 *pu64Knights,
                      OUT UINT64 *pu64Pawns)
/**

Routine description:

    Build the set of bitboards Fathom's tb_probe_wdl wants out of our
    POSITION.  Fathom expects the standard tablebase square numbering
    (a1 == bit 0 ... h8 == bit 63); our own internal bitboards
    (bbPieces/bbPawns/bbOccupiedSide) instead index bit = rankTopDown*8
    + file (rankTopDown 0 == rank 8 ... 7 == rank 1 -- see TO64's own
    (7 - (x>>4)) flip). Those two numberings differ by exactly one
    64-bit byte-swap: internal bit r*8+f (byte r, offset f) lands at
    Fathom bit (7-r)*8+f (byte 7-r, offset f) under bswap64, matching
    TO64's formula term for term. So this can OR our own bitboards
    together and byte-swap the result directly -- O(1) per output
    bitboard -- instead of walking every piece one at a time through
    CoorFromBitBoardRank8ToRank1/TO64. (2026-09-24: replaced the
    original per-piece-popping version, which showed up as a real
    self-play NPS/depth regression once EgtbProbeLikelyResident started
    calling this from QSearch's hot path -- see CLAUDE.md.)

    No bbPieces[color][KING] exists (deliberately -- see its own
    comment), so the king bitboard is built from the single square in
    cKingSq[color] instead; equivalent to bswapping a bitboard with only
    that one bit set, since bswap64 is just a bit permutation and
    commutes with OR.

Parameters:

    POSITION *pos,
    UINT64 *pu64White ... pu64Pawns

Return value:

    static void

**/
{
    UINT64 u64Kings = (1ULL << COOR_TO_BIT_NUMBER(pos->cKingSq[WHITE])) |
                       (1ULL << COOR_TO_BIT_NUMBER(pos->cKingSq[BLACK]));

    *pu64White = __builtin_bswap64((UINT64)pos->bbOccupiedSide[WHITE]);
    *pu64Black = __builtin_bswap64((UINT64)pos->bbOccupiedSide[BLACK]);
    *pu64Kings = __builtin_bswap64(u64Kings);
    *pu64Queens = __builtin_bswap64(
        (UINT64)(pos->bbPieces[WHITE][QUEEN] | pos->bbPieces[BLACK][QUEEN]));
    *pu64Rooks = __builtin_bswap64(
        (UINT64)(pos->bbPieces[WHITE][ROOK] | pos->bbPieces[BLACK][ROOK]));
    *pu64Bishops = __builtin_bswap64(
        (UINT64)(pos->bbPieces[WHITE][BISHOP] | pos->bbPieces[BLACK][BISHOP]));
    *pu64Knights = __builtin_bswap64(
        (UINT64)(pos->bbPieces[WHITE][KNIGHT] | pos->bbPieces[BLACK][KNIGHT]));
    *pu64Pawns = __builtin_bswap64(
        (UINT64)(pos->bbPawns[WHITE] | pos->bbPawns[BLACK]));
}


FLAG
ProbeEGTB(SEARCHER_THREAD_CONTEXT *ctx,
          SCORE *piScore)
/**

Routine description:

    Search for a board position in the Syzygy EGTB files on disk via
    Fathom.  This only returns a WDL (win/draw/loss) result, not a
    literal mate distance -- see EGTB_WIN_SCORE / EGTB_LOSS_SCORE.

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx,
    SCORE *piScore

Return value:

    FLAG

**/
{
    POSITION *pos = &(ctx->sPosition);
    UINT64 u64White, u64Black, u64Kings, u64Queens;
    UINT64 u64Rooks, u64Bishops, u64Knights, u64Pawns;
    unsigned uEp;
    unsigned uWdl;
    ULONG wcount, bcount;
    FLAG fResult;

    if (FALSE == g_fEgtbInitialized || 0 == TB_LARGEST)
    {
        return(FALSE);
    }

    //
    // Syzygy WDL probing requires no castling rights on the board and
    // a zero halfmove (fifty move rule) clock; the WDL tables don't
    // encode either.
    //
    if ((0 != pos->bvCastleInfo) || (0 != pos->uFifty))
    {
        return(FALSE);
    }

    wcount = TotalNonPawnCount(pos, WHITE) + FastCountBits(pos->bbPawns[WHITE]);
    bcount = TotalNonPawnCount(pos, BLACK) + FastCountBits(pos->bbPawns[BLACK]);
    if ((wcount + bcount) > TB_LARGEST)
    {
        return(FALSE);
    }
    INC(ctx->sCounters.egtb.uProbes);

    _BuildFathomBitboards(pos, &u64White, &u64Black, &u64Kings, &u64Queens,
                          &u64Rooks, &u64Bishops, &u64Knights, &u64Pawns);

    uEp = 0;
    if (IS_ON_BOARD(pos->cEpSquare))
    {
        uEp = TO64(pos->cEpSquare);
    }

    AcquireSpinLock(&g_uEgtbLock);
    uWdl = tb_probe_wdl(u64White, u64Black, u64Kings, u64Queens, u64Rooks,
                        u64Bishops, u64Knights, u64Pawns,
                        /* rule50 */ 0, /* castling */ 0, uEp,
                        (pos->uToMove == WHITE));
    ReleaseSpinLock(&g_uEgtbLock);

    if (TB_RESULT_FAILED == uWdl)
    {
        return(FALSE);
    }

    switch (uWdl)
    {
        case TB_WIN:
            *piScore = EGTB_WIN_SCORE;
            break;
        case TB_LOSS:
            *piScore = EGTB_LOSS_SCORE;
            break;
        case TB_CURSED_WIN:
        case TB_BLESSED_LOSS:
        case TB_DRAW:
        default:
            *piScore = g_iDrawScore[pos->uToMove];
            break;
    }
    fResult = TRUE;

#ifdef PERF_COUNTERS
    INC(ctx->sCounters.egtb.uHits);
#endif
    return(fResult);
}


FLAG
EgtbProbeLikelyResident(SEARCHER_THREAD_CONTEXT *ctx)
/**

Routine description:

    Cheap, non-blocking predictor for "would an EGTB probe of the
    current position be low latency?" -- used to let deep search/qsearch
    nodes probe the EGTB (which is normally reserved for shallow, near-
    root plies to avoid stalling the tree search on a disk read) once
    the specific pages this exact position's lookup needs are already
    resident, whether from an earlier probe this run or (for 3-5 man
    tables) because they were mlock()'d in whole at load time.

    Does not itself probe the tablebase, take g_uEgtbLock, or read any
    file data that isn't already confirmed resident first -- see
    tb_wdl_probe_is_resident in fathom/tbprobe.c for the mincore()-gated
    walk this performs (indexTable/sizeTable/data-block addresses for
    6-man tables; a single ready[] check for the mlock()'d 3-5 man
    tier). Safe to call from every leaf-ish node without worrying about
    search slowdown.

    Same castling/fifty-move/cardinality gating as ProbeEGTB (a probe
    that ProbeEGTB itself would refuse is never "likely resident").

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx

Return value:

    FLAG -- TRUE iff a subsequent ProbeEGTB call for this exact position
    is expected to hit already-resident tablebase data rather than pay a
    first-touch mmap()/parse (and possibly disk-read) cost.

**/
{
    POSITION *pos = &(ctx->sPosition);
    UINT64 u64White, u64Black, u64Kings, u64Queens;
    UINT64 u64Rooks, u64Bishops, u64Knights, u64Pawns;
    ULONG wcount, bcount;

    if (FALSE == g_fEgtbInitialized || 0 == TB_LARGEST)
    {
        return(FALSE);
    }
    if ((0 != pos->bvCastleInfo) || (0 != pos->uFifty))
    {
        return(FALSE);
    }
    wcount = TotalNonPawnCount(pos, WHITE) + FastCountBits(pos->bbPawns[WHITE]);
    bcount = TotalNonPawnCount(pos, BLACK) + FastCountBits(pos->bbPawns[BLACK]);
    if ((wcount + bcount) > TB_LARGEST)
    {
        return(FALSE);
    }

    _BuildFathomBitboards(pos, &u64White, &u64Black, &u64Kings, &u64Queens,
                          &u64Rooks, &u64Bishops, &u64Knights, &u64Pawns);
    return(1 == tb_wdl_probe_is_resident(u64White, u64Black, u64Kings,
                                          u64Queens, u64Rooks, u64Bishops,
                                          u64Knights, u64Pawns,
                                          (pos->uToMove == WHITE)));
}


//
// Convert a Fathom tablebase square index (a1==0 ... h8==63) back into
// our own COOR encoding.  Inverse of TO64().
//
#define FROM64(s)                   \
    ((((7 - ((s) >> 3)) << 4)) | ((s) & 0x7))


static PIECE
_TbPromotesToPiece(IN unsigned uPromotes, IN ULONG uSide)
/**

Routine description:

    Map a TB_PROMOTES_* value from a Fathom root probe result to our
    own PIECE constant. Must return a *colored* piece (BLACK_QUEEN|
    uSide, matching san.c's "pPromoted = BLACK_QUEEN | pos->uToMove"
    convention -- WHITE=1/BLACK=0 ORed onto the black-piece base value
    selects the right color bit) since that's what move generation
    actually stamps into mv.pPromoted. The previous version returned
    the bare, colorless QUEEN/ROOK/BISHOP/KNIGHT constants (5/3/2/4 --
    type-only values, never equal to any real generated move's
    pPromoted), so ProbeEGTBRoot's move-list lookup below could never
    match a promotion move for either side and always fell through to
    ASSERT(FALSE) -- 100% reproducible whenever Fathom's suggested
    root DTZ move happened to be a promotion, rare enough to need a
    200k-position stress run to surface.

Parameters:

    unsigned uPromotes
    ULONG uSide : WHITE or BLACK -- the side making the promoting move

Return value:

    static PIECE

**/
{
    switch (uPromotes)
    {
        case TB_PROMOTES_QUEEN:  return(BLACK_QUEEN  | uSide);
        case TB_PROMOTES_ROOK:   return(BLACK_ROOK   | uSide);
        case TB_PROMOTES_BISHOP: return(BLACK_BISHOP | uSide);
        case TB_PROMOTES_KNIGHT: return(BLACK_KNIGHT | uSide);
        default:                 return(0);
    }
}


FLAG
ProbeEGTBRoot(SEARCHER_THREAD_CONTEXT *ctx,
              MOVE *pmv,
              SCORE *piScore)
/**

Routine description:

    Probe the Syzygy DTZ tables for the best move at the root.  Unlike
    ProbeEGTB (WDL-only, used at interior nodes), this uses DTZ to
    pick a move that actually makes progress towards mate / the
    fifty-move-rule reset -- Syzygy WDL alone has no notion of "closer
    to mate" so relying on it exclusively could shuffle forever inside
    a won position.  This is expected to be called once at the root,
    before the normal iterative search, when <= TB_LARGEST pieces
    remain on the board.

    We ask Fathom for its suggested (from, to, promotion) and then
    find the matching move in our own already-generated root move
    list, rather than reconstructing a MOVE by hand, so we inherit our
    engine's own move flags (capture, en passant, special, etc).

Parameters:

    SEARCHER_THREAD_CONTEXT *ctx,
    MOVE *pmv,
    SCORE *piScore

Return value:

    FLAG : TRUE if a move was found and *pmv / *piScore were set.

**/
{
    POSITION *pos = &(ctx->sPosition);
    UINT64 u64White, u64Black, u64Kings, u64Queens;
    UINT64 u64Rooks, u64Bishops, u64Knights, u64Pawns;
    unsigned uEp;
    unsigned uResult;
    unsigned uWdl;
    COOR cFrom, cTo;
    PIECE pPromotes;
    ULONG wcount, bcount;
    ULONG u;

    if (FALSE == g_fEgtbInitialized || 0 == TB_LARGEST)
    {
        return(FALSE);
    }

    wcount = TotalNonPawnCount(pos, WHITE) + FastCountBits(pos->bbPawns[WHITE]);
    bcount = TotalNonPawnCount(pos, BLACK) + FastCountBits(pos->bbPawns[BLACK]);
    if ((wcount + bcount) > TB_LARGEST)
    {
        return(FALSE);
    }

    _BuildFathomBitboards(pos, &u64White, &u64Black, &u64Kings, &u64Queens,
                          &u64Rooks, &u64Bishops, &u64Knights, &u64Pawns);

    uEp = 0;
    if (IS_ON_BOARD(pos->cEpSquare))
    {
        uEp = TO64(pos->cEpSquare);
    }

    AcquireSpinLock(&g_uEgtbLock);
    uResult = tb_probe_root(u64White, u64Black, u64Kings, u64Queens, u64Rooks,
                            u64Bishops, u64Knights, u64Pawns,
                            pos->uFifty, pos->bvCastleInfo, uEp,
                            (pos->uToMove == WHITE), NULL);
    ReleaseSpinLock(&g_uEgtbLock);

    if ((TB_RESULT_FAILED == uResult) ||
        (TB_RESULT_CHECKMATE == uResult) ||
        (TB_RESULT_STALEMATE == uResult))
    {
        //
        // No usable move (either failed, or the earlier terminal-state
        // check already handles checkmate/stalemate).
        //
        return(FALSE);
    }

    //
    // Always follow Fathom's root DTZ suggestion once the tablebase has
    // a verdict, including plain draws and the cursed-win/blessed-loss
    // buckets.  A cursed win's/blessed loss's DTZ move is still the one
    // that makes maximum progress toward actually converting/surviving
    // before the fifty-move counter forces a draw -- ordinary search,
    // steered only by a heuristic eval with no notion of the tablebase's
    // verdict, can and does wander off the single correct try in these
    // cases (confirmed live: a blessed-loss position in blunders.pgn
    // where unassisted search picked a move that lost outright). Plain
    // TB_DRAW has many equally-drawing replies, so this is never worse
    // than deferring to search, just occasionally no better.
    //
    uWdl = TB_GET_WDL(uResult);

    cFrom = FROM64(TB_GET_FROM(uResult));
    cTo = FROM64(TB_GET_TO(uResult));
    pPromotes = _TbPromotesToPiece(TB_GET_PROMOTES(uResult), pos->uToMove);

    for (u = ctx->sMoveStack.uBegin[ctx->uPly];
         u < ctx->sMoveStack.uEnd[ctx->uPly];
         u++)
    {
        MOVE mv = ctx->sMoveStack.mvf[u].mv;
        if ((mv.cFrom == cFrom) &&
            (mv.cTo == cTo) &&
            (mv.pPromoted == pPromotes))
        {
            // DTZ (distance to the next fifty-move-rule-zeroing move)
            // is not the same thing as distance to mate -- a previous
            // version of this code reported (INFINITY_SCORE -/+ uDtz)
            // as a mate score, which is wrong in general: DTZ can be
            // small in a dead-won position that's still many plies
            // from actual mate (e.g. right as pos->uFifty climbs
            // towards 100, a reset becomes forced soon even though
            // mate itself is nowhere near), producing a bogus
            // "+MATE<n>"/"-MATE<n>" display untethered from reality.
            // Report the same flat mate-magnitude score ProbeEGTB/
            // RecognLookup use at interior nodes for a clear win/loss;
            // the drawish buckets (plain draw, cursed win, blessed
            // loss) get the normal draw score instead, matching
            // ProbeEGTB's own WDL->score mapping.
            *pmv = mv;
            switch (uWdl)
            {
                case TB_WIN:  *piScore = +NMATE; break;
                case TB_LOSS: *piScore = -NMATE; break;
                default:      *piScore = g_iDrawScore[pos->uToMove]; break;
            }
            return(TRUE);
        }
    }

    //
    // Fathom suggested a move we couldn't find in our own move list;
    // this shouldn't happen for a legally-generated root, but don't
    // trust it if it does.
    //
    ASSERT(FALSE);
    return(FALSE);
}