path: root/src/testrecogn.c

/**

Copyright (c) Scott Gasch

Module Name:

    testrecogn.c

Abstract:

    Cross-check the interior-node recognizers in recogn.c against
    Syzygy EGTB ground truth (via ProbeEGTB/fathom).  For each
    registered recognizer material signature, build a batch of random
    legal positions matching that exact signature, ask RecognLookup
    for a recognizer-only verdict (fProbeEGTB == FALSE, so a hit is
    guaranteed to have come from hand-written recognizer logic, not a
    table probe), then verify the verdict against a direct ProbeEGTB
    call on the same position whenever tablebase coverage is
    available.  This only proves anything when the box actually has
    EGTB files installed and initialized (see CLAUDE.md's note on
    /zscratch/egtb) -- if not, this still exercises RecognLookup/the
    position builder for crashes but silently skips the real
    cross-check for every position (reported in the trailing summary
    line so that's visible rather than a quiet false pass).

Author:

    Scott Gasch ([email protected])

Revision History:

**/

#include "chess.h"

#ifdef TEST

#define RECOGN_TEST_ITERATIONS_PER_CASE (150)

typedef struct _RECOGN_CASE
{
    const char *szName;
    ULONG uPawnLoA, uPawnHiA;
    ULONG uKnightLoA, uKnightHiA;
    ULONG uBishopLoA, uBishopHiA;
    ULONG uPawnLoB, uPawnHiB;
    ULONG uKnightLoB, uKnightHiB;
    ULONG uBishopLoB, uBishopHiB;
} RECOGN_CASE;

//
// One entry per material signature pair registered in
// InitializeInteriorNodeRecognizers (recogn.c).  "A"/"B" are abstract
// slots, randomly assigned to WHITE/BLACK per generated position (see
// _MakeRandomEndgame) so both color orderings get exercised.
//
static RECOGN_CASE g_rgRecognCases[] =
{
    // name       pawnA        knightA      bishopA       pawnB        knightB      bishopB
    { "KK",       0, 0,        0, 0,        0, 0,         0, 0,        0, 0,        0, 0 },
    { "KBK",      0, 0,        0, 0,        1, 2,         0, 0,        0, 0,        0, 0 },
    { "KBKB",     0, 0,        0, 0,        1, 2,         0, 0,        0, 0,        1, 2 },
    { "KNK",      0, 0,        1, 2,        0, 0,         0, 0,        0, 0,        0, 0 },
    { "KNKN",     0, 0,        1, 2,        0, 0,         0, 0,        1, 2,        0, 0 },
    { "KNKB",     0, 0,        1, 2,        0, 0,         0, 0,        0, 0,        1, 2 },
    { "KNBK",     0, 0,        1, 1,        1, 1,         0, 0,        0, 0,        0, 0 },
    { "KNKP",     1, 3,        0, 0,        0, 0,         0, 0,        1, 2,        0, 0 },
    { "KBKP",     1, 3,        0, 0,        0, 0,         0, 0,        0, 0,        1, 2 },
    { "KPBKP",    1, 2,        0, 0,        1, 2,         1, 2,        0, 0,        0, 0 },
    { "KPBK",     1, 2,        0, 0,        1, 2,         0, 0,        0, 0,        0, 0 },
    { "KPK",      0, 0,        0, 0,        0, 0,         1, 3,        0, 0,        0, 0 },
    { "KPKP",     1, 3,        0, 0,        0, 0,         1, 3,        0, 0,        0, 0 },
};

#define RECOGN_RANGE(lo, hi) \
    ((lo) + (ULONG)(rand() % (((hi) - (lo)) + 1)))

//
// Build one random position with exactly uPawnA/uKnightA/uBishopA
// pieces of those types for a randomly-picked color and
// uPawnB/uKnightB/uBishopB for the other color, plus two kings.
// Returns FALSE (caller should retry) if the random placement turned
// out illegal (most commonly: a slider placed such that the side not
// on move is left in check).
//
static FLAG
_TryBuildRandomEndgame(OUT POSITION *pos,
                       IN ULONG uPawnA, IN ULONG uKnightA, IN ULONG uBishopA,
                       IN ULONG uPawnB, IN ULONG uKnightB, IN ULONG uBishopB)
{
    PIECE pBoard[128];
    CHAR szFen[256];
    CHAR *p;
    COOR c, cWhiteKing, cBlackKing;
    ULONG uColorA, uColorB;
    ULONG uPawn[2], uKnight[2], uBishop[2];
    ULONG uColor, u, x;

    memset(pBoard, 0, sizeof(pBoard));

    uColorA = RANDOM_COLOR;
    uColorB = FLIP(uColorA);
    uPawn[uColorA] = uPawnA;
    uPawn[uColorB] = uPawnB;
    uKnight[uColorA] = uKnightA;
    uKnight[uColorB] = uKnightB;
    uBishop[uColorA] = uBishopA;
    uBishop[uColorB] = uBishopB;

    do
    {
        cWhiteKing = RANDOM_COOR;
    }
    while (!IS_ON_BOARD(cWhiteKing));
    do
    {
        cBlackKing = RANDOM_COOR;
    }
    while ((!IS_ON_BOARD(cBlackKing)) ||
           (DISTANCE(cWhiteKing, cBlackKing) < 2));
    pBoard[cWhiteKing] = WHITE_KING;
    pBoard[cBlackKing] = BLACK_KING;

    FOREACH_COLOR(uColor)
    {
        for (u = 0; u < uPawn[uColor]; u++)
        {
            do
            {
                c = RANDOM_COOR;
            }
            while ((!IS_ON_BOARD(c)) || (0 != pBoard[c]) ||
                   RANK1(c) || RANK8(c));
            pBoard[c] = (WHITE == uColor) ? WHITE_PAWN : BLACK_PAWN;
        }
        for (u = 0; u < uKnight[uColor]; u++)
        {
            do
            {
                c = RANDOM_COOR;
            }
            while ((!IS_ON_BOARD(c)) || (0 != pBoard[c]));
            pBoard[c] = (WHITE == uColor) ? WHITE_KNIGHT : BLACK_KNIGHT;
        }
        for (u = 0; u < uBishop[uColor]; u++)
        {
            do
            {
                c = RANDOM_COOR;
            }
            while ((!IS_ON_BOARD(c)) || (0 != pBoard[c]));
            pBoard[c] = (WHITE == uColor) ? WHITE_BISHOP : BLACK_BISHOP;
        }
    }

    memset(szFen, 0, sizeof(szFen));
    p = szFen;
    for (x = 0; x < 120; x++)
    {
        if ((x % 15 == 0) && (x != 0))
        {
            *p++ = '/';
        }
        if (!IS_ON_BOARD(x)) continue;
        if (0 == pBoard[x])
        {
            *p++ = '1';
        }
        else
        {
            *p++ = *(PieceAbbrev(pBoard[x]) + 1);
        }
    }
    *p++ = ' ';
    *p++ = (RANDOM_COLOR == WHITE) ? 'w' : 'b';
    strcat(p, " - - 0 1");

    if (FALSE == FenToPosition(pos, szFen))
    {
        return(FALSE);
    }
    if (InCheck(pos, FLIP(pos->uToMove)))
    {
        return(FALSE);
    }
    return(TRUE);
}

static void
_MakeRandomEndgame(OUT POSITION *pos,
                    IN ULONG uPawnA, IN ULONG uKnightA, IN ULONG uBishopA,
                    IN ULONG uPawnB, IN ULONG uKnightB, IN ULONG uBishopB)
{
    while (FALSE == _TryBuildRandomEndgame(pos, uPawnA, uKnightA, uBishopA,
                                            uPawnB, uKnightB, uBishopB))
    {
        // retry with a fresh random placement
    }
}

//
// Does a recognizer's returned bound/exact verdict (iRecScore, uVal --
// both relative to the position's side to move, same convention as
// ProbeEGTB) agree with a coarse WDL-only EGTB probe (iEgtbScore, also
// relative to side to move)?  This mirrors the corrected reasoning
// used in recogn.c's own (DEBUG-only) _SanityCheckRecognizers: a
// LOWER/UPPER bound only licenses a directional claim on the side of
// the bound that's actually pinned down (a positive lower bound forces
// a real win; a negative lower bound makes no claim at all), not the
// mirror-image claim a copy/paste of the EXACT case would produce.
//
static FLAG
_RecognizerAgreesWithEGTB(IN SCORE iRecScore,
                          IN ULONG uVal,
                          IN SCORE iEgtbScore)
{
    switch (uVal)
    {
        case RECOGN_EXACT:
            if (iRecScore == 0) return(iEgtbScore == 0);
            if (iRecScore > 0) return(iEgtbScore > 0);
            return(iEgtbScore < 0);

        case RECOGN_LOWER:
            if (iRecScore > 0) return(iEgtbScore > 0);
            if (iRecScore == 0) return(iEgtbScore >= 0);
            return(TRUE);

        case RECOGN_UPPER:
            if (iRecScore < 0) return(iEgtbScore < 0);
            if (iRecScore == 0) return(iEgtbScore <= 0);
            return(TRUE);

        default:
            return(TRUE);
    }
}

void
TestRecogn(void)
/**

Routine description:

    Entry point: build random positions for every registered
    recognizer material signature and cross-check RecognLookup's
    verdict against direct EGTB probes wherever coverage exists.

Parameters:

    void

Return value:

    void

**/
{
    SEARCHER_THREAD_CONTEXT *ctx;
    POSITION pos;
    ULONG uCase, uIter;
    ULONG uPawnA, uKnightA, uBishopA, uPawnB, uKnightB, uBishopB;
    SCORE iScore, iEgtb;
    ULONG uVal;
    ULONG uChecked = 0, uSkippedNoEgtb = 0, uUnrecognized = 0;
    RECOGN_CASE *p;

    Trace("Testing interior-node recognizers against EGTB ground truth...\n");
    ctx = malloc(sizeof(SEARCHER_THREAD_CONTEXT));
    if (NULL == ctx)
    {
        return;
    }
    //
    // InitializeSearcherContext memsets the whole (tens-of-MB)
    // SEARCHER_THREAD_CONTEXT -- fine once, catastrophic per-iteration
    // in a loop like this one (same pitfall CLAUDE.md documents for
    // the old EvalCommand bug). Init once, then use the cheap
    // ReInitializeSearcherContext (just repositions + zeros counters)
    // per generated position below.
    //
    InitializeSearcherContext(&pos, ctx);

    for (uCase = 0; uCase < ARRAY_LENGTH(g_rgRecognCases); uCase++)
    {
        p = &g_rgRecognCases[uCase];
        for (uIter = 0; uIter < RECOGN_TEST_ITERATIONS_PER_CASE; uIter++)
        {
            uPawnA = RECOGN_RANGE(p->uPawnLoA, p->uPawnHiA);
            uKnightA = RECOGN_RANGE(p->uKnightLoA, p->uKnightHiA);
            uBishopA = RECOGN_RANGE(p->uBishopLoA, p->uBishopHiA);
            uPawnB = RECOGN_RANGE(p->uPawnLoB, p->uPawnHiB);
            uKnightB = RECOGN_RANGE(p->uKnightLoB, p->uKnightHiB);
            uBishopB = RECOGN_RANGE(p->uBishopLoB, p->uBishopHiB);

            _MakeRandomEndgame(&pos, uPawnA, uKnightA, uBishopA,
                               uPawnB, uKnightB, uBishopB);
            ReInitializeSearcherContext(&pos, ctx);

            uVal = RecognLookup(ctx, &iScore, FALSE);
            if (UNRECOGNIZED == uVal)
            {
                uUnrecognized++;
                continue;
            }

            if ((iScore <= -NMATE) || (iScore >= NMATE))
            {
                UtilPanic(TESTCASE_FAILURE, &pos,
                          (void *)p->szName,
                          "recognizer score out of range",
                          NULL, __FILE__, __LINE__);
            }

            if (FALSE == ProbeEGTB(ctx, &iEgtb))
            {
                uSkippedNoEgtb++;
                continue;
            }
            uChecked++;

            if (FALSE == _RecognizerAgreesWithEGTB(iScore, uVal, iEgtb))
            {
                UtilPanic(TESTCASE_FAILURE, &pos,
                          (void *)p->szName,
                          "recognizer verdict disagrees with EGTB",
                          NULL, __FILE__, __LINE__);
            }
        }
    }

    free(ctx);
    Trace("Recognizer/EGTB cross-check: %lu positions verified, "
          "%lu skipped (no EGTB coverage), %lu unrecognized.\n",
          uChecked, uSkippedNoEgtb, uUnrecognized);
    if (0 == uChecked)
    {
        Trace("WARNING: zero positions were actually cross-checked "
              "against EGTB -- no tablebase coverage available, so "
              "this run only smoke-tested RecognLookup, not its "
              "correctness.\n");
    }
}

//
// Build a KNKP (one knight, one pawn) position directly from square
// placements -- white always plays the knight, black always plays the
// pawn (color-symmetric by construction: _RecognizeKNKP determines
// "strong side" dynamically from piece counts, not from hardcoded
// color, so fixing white=knight loses no coverage and halves the
// enumeration).
//
static FLAG
_BuildKNKPPosition(OUT POSITION *pos,
                   IN COOR cStrongKing, IN COOR cWeakKing,
                   IN COOR cKnight, IN COOR cPawn,
                   IN ULONG uToMove)
{
    PIECE pBoard[128];
    CHAR szFen[256];
    CHAR *p;
    ULONG x;

    memset(pBoard, 0, sizeof(pBoard));
    pBoard[cStrongKing] = WHITE_KING;
    pBoard[cWeakKing] = BLACK_KING;
    pBoard[cKnight] = WHITE_KNIGHT;
    pBoard[cPawn] = BLACK_PAWN;

    memset(szFen, 0, sizeof(szFen));
    p = szFen;
    for (x = 0; x < 120; x++)
    {
        if ((x % 15 == 0) && (x != 0))
        {
            *p++ = '/';
        }
        if (!IS_ON_BOARD(x)) continue;
        if (0 == pBoard[x])
        {
            *p++ = '1';
        }
        else
        {
            *p++ = *(PieceAbbrev(pBoard[x]) + 1);
        }
    }
    *p++ = ' ';
    *p++ = (WHITE == uToMove) ? 'w' : 'b';
    strcat(p, " - - 0 1");

    if (FALSE == FenToPosition(pos, szFen))
    {
        return(FALSE);
    }
    if (InCheck(pos, FLIP(pos->uToMove)))
    {
        return(FALSE);
    }
    return(TRUE);
}

void
TestRecognExhaustiveKNKP(void)
/**

Routine description:

    Exhaustively enumerate every legal KNKP (one knight, one pawn)
    position -- not a random sample -- and verify _RecognizeKNKP's
    verdict against direct EGTB probes wherever coverage exists. This
    is the acceptance bar a fixed/rewritten KNKP recognizer should meet
    before ever being re-registered in
    InitializeInteriorNodeRecognizers: "no counterexamples in N random
    samples" is exactly the false confidence that let the original
    KNKP recognizer ship with two independent, real losing lines (see
    the comment on _RecognizeKNKP in recogn.c). This enumeration uses
    left/right board mirror symmetry (pawns break rank symmetry, not
    file symmetry) to roughly halve the raw square-placement count, and
    fixes white=knight/black=pawn (also lossless -- see
    _BuildKNKPPosition) -- what's left is still several million raw
    square combinations, so this is deliberately NOT wired into
    TestRecogn()'s automatic self-test sequence (that runs on every
    engine startup and is expected to stay fast, per CLAUDE.md's
    precommit_check.sh budget). Call this on demand instead when
    actually validating a KNKP fix.

Parameters:

    void

Return value:

    void

**/
{
    SEARCHER_THREAD_CONTEXT *ctx;
    POSITION pos;
    COOR cStrongKing, cWeakKing, cKnight, cPawn;
    ULONG uToMove;
    SCORE iScore, iEgtb;
    ULONG uVal;
    ULONG uTotal = 0, uChecked = 0, uSkipped = 0, uWrong = 0;
    ULONG uReportedWrong = 0;

    Trace("Exhaustive KNKP (1 knight, 1 pawn) verification against "
          "EGTB ground truth -- this takes a while...\n");
    ctx = malloc(sizeof(SEARCHER_THREAD_CONTEXT));
    if (NULL == ctx)
    {
        return;
    }
    //
    // Same fix as TestRecogn above: init the (tens-of-MB) context once,
    // not once per generated position -- at millions of iterations the
    // per-call memset inside InitializeSearcherContext dominates
    // runtime completely (this is what made the first attempt at this
    // function never finish within a 10-minute budget).
    //
    InitializeSearcherContext(&pos, ctx);

    FOREACH_SQUARE(cStrongKing)
    {
        if (!IS_ON_BOARD(cStrongKing)) continue;
        if (FILE(cStrongKing) > D) continue;

        FOREACH_SQUARE(cWeakKing)
        {
            if (!IS_ON_BOARD(cWeakKing)) continue;
            if (DISTANCE(cStrongKing, cWeakKing) < 2) continue;

            FOREACH_SQUARE(cKnight)
            {
                if (!IS_ON_BOARD(cKnight)) continue;
                if ((cKnight == cStrongKing) || (cKnight == cWeakKing)) continue;

                FOREACH_SQUARE(cPawn)
                {
                    if (!IS_ON_BOARD(cPawn)) continue;
                    if (RANK1(cPawn) || RANK8(cPawn)) continue;
                    if ((cPawn == cStrongKing) || (cPawn == cWeakKing) ||
                        (cPawn == cKnight)) continue;

                    for (uToMove = BLACK; uToMove <= WHITE; uToMove++)
                    {
                        uTotal++;
                        if (0 == (uTotal % 500000))
                        {
                            Trace("  ... %lu raw combos so far (%lu checked, "
                                  "%lu wrong)\n", uTotal, uChecked, uWrong);
                        }
                        if (FALSE == _BuildKNKPPosition(&pos, cStrongKing,
                                                         cWeakKing, cKnight,
                                                         cPawn, uToMove))
                        {
                            continue;
                        }
                        ReInitializeSearcherContext(&pos, ctx);
                        uVal = _RecognizeKNKP(ctx, &iScore);
                        if (UNRECOGNIZED == uVal)
                        {
                            continue;
                        }
                        if (FALSE == ProbeEGTB(ctx, &iEgtb))
                        {
                            uSkipped++;
                            continue;
                        }
                        uChecked++;
                        if (FALSE == _RecognizerAgreesWithEGTB(iScore, uVal,
                                                                iEgtb))
                        {
                            uWrong++;
                            if (uReportedWrong < 10)
                            {
                                Trace("MISMATCH iScore=%d uVal=%lu "
                                      "iEgtb=%d toMove=%lu\n",
                                      iScore, uVal, iEgtb, pos.uToMove);
                                DumpPosition(&pos);
                                uReportedWrong++;
                            }
                        }
                    }
                }
            }
        }
    }

    free(ctx);
    Trace("Exhaustive KNKP: %lu raw combos, %lu checked against EGTB, "
          "%lu skipped (no EGTB coverage), %lu WRONG.\n",
          uTotal, uChecked, uSkipped, uWrong);
}

#endif // TEST