Two ideas:

1. Nullmove R=3 + depth/6 along with some ShouldWeDoNullmoveSearch
tweaking and calibrating.

2. Probe EGTB everywhere in the search as long as the table/position
is resident in memory.

1 files changed, 304 insertions, 5 deletions

diff --git a/src/search.c b/src/search.c
index 0362701..8f4779e 100644
--- a/src/search.c
+++ b/src/search.c
@@ -104,6 +104,131 @@ _CMEvidenceBucket(ULONG uEvidence)
 }
 #endif
 
+#ifdef TEST_NULL
+//
+// Null-move try/avoid calibration harness.
+//
+// Question: given the current data (precision >99.9% almost everywhere,
+// false-positive/"null-move lie" risk concentrated by material and
+// deficit, EV strongly positive almost everywhere once restricted to
+// the safe region), can a two-stage try/skip policy do better than
+// today's flat material threshold + per-depth eval-margin table?
+//
+//   Stage 1 (safety): is a null-move fail-high here even trustworthy?
+//   Stage 2 (profitability/EV): given it's safe, is it worth the cost?
+//
+// fThreat, fSkipNull, and en-prise/trapped-piece state were bucketed in
+// an earlier revision of this harness (2026-09-24 run, saved under
+// nullmove_calib/run_2026-09-24_sd10_30k/) and shown to be either
+// already-validated non-tunable binary gates (fThreat/fSkipNull -- both
+// clearly reduce EV and precision/recall when set, and both already
+// gate the call to WeShouldTryNullmovePruning in Search(), so there's
+// no threshold left to calibrate) or not a safety signal at all
+// (en-prise/trapped predicts *lower recall*, not higher false-positive
+// risk -- a Stage 2, not Stage 1, concern). None of the three are
+// bucketed here; fThreat==FALSE && fSkipNull==FALSE are folded into
+// this probe's structural eligibility instead, matching production.
+//
+// This revision instead resolves two open questions from that first
+// run:
+//
+//   1. Is MAX(white,black) non-pawn piece count or the MOVER's own
+//      count (TotalNonPawnCount(pos, pos->uToMove)) the better
+//      Stage-1 safety predictor? Zugzwang is a property of the side
+//      being given the free "pass", so mover-only is the more
+//      causally-motivated signal, but the first run only measured
+//      MAX(both) -- bucket both, jointly, and let the false-positive-
+//      rate data settle it.
+//   2. The first run bucketed the *real* search's cost (the Stage-2 EV
+//      calc's "cost of the alternative") by the null probe's own post-
+//      reduction depth (uNullDepth), which is only meaningful via
+//      whatever R schedule (SelectNullmoveRFactor) happened to be
+//      compiled in at the time -- useless for evaluating a *different*
+//      R schedule later. The real search's cost is fundamentally a
+//      function of the OUTER, pre-reduction uDepth (how much real work
+//      remains), not of uNullDepth at all. Bucket both depths
+//      separately: uNullDepth for the probe's own cost (cost1), uDepth
+//      for the real alternative's cost (cost2) -- decoupling the two
+//      cost models from any particular R formula.
+//
+// Ground truth (did this node *really* fail high) is read off the same
+// iBestScore-vs-iBeta comparison Search() already computes at its
+// "end:" label for the NODE_PV/NODE_CUT/NODE_ALL surprise counters just
+// above this block; we piggyback on that rather than duplicating it.
+// The diagnostic probe is a self-contained MakeMove(NULLMOVE) -> reduced
+// Search() -> UnmakeMove, deliberately *not* reusing TryNullmovePruning,
+// so its verification-search/extension side effects (meant only for a
+// probe whose result production actually trusts) never fire here. Like
+// any double-search diagnostic, it also warms the hash/history tables
+// along its own subtree before the real move loop runs at the same
+// node -- a known, unavoidable side effect of measuring this way.
+//
+// This roughly doubles node cost at every non-PV node under TEST_NULL,
+// so it's a diagnostic-only build flag (gmake TEST_NULL=1 ...), never
+// part of the normal GENETIC=1 PERF_COUNTERS=1 MP=1 SIXTYFOUR=1 profile,
+// and should only be run against a calibration corpus at a shallow
+// fixed depth, not self-play or the full engine pools.
+//
+// Buckets are a mixed-radix index over five dimensions:
+//
+//   index = nullDepthB  * (NC_DEPTH_BUCKETS*NC_DEFICIT_BUCKETS*NC_MATERIAL_BUCKETS*NC_MATERIAL_BUCKETS)
+//         + outerDepthB * (NC_DEFICIT_BUCKETS*NC_MATERIAL_BUCKETS*NC_MATERIAL_BUCKETS)
+//         + deficitB    * (NC_MATERIAL_BUCKETS*NC_MATERIAL_BUCKETS)
+//         + maxMaterialB* (NC_MATERIAL_BUCKETS)
+//         + moverMaterialB
+//
+// root.c's dump loop decodes this same layout when printing results.
+//
+static ULONG
+_NullCalibDepthBucket(ULONG uSomeDepth)  // generic: used for both uNullDepth and uDepth
+{
+    ULONG uPlies = uSomeDepth / ONE_PLY;
+    if (uPlies >= NC_DEPTH_BUCKETS - 1) return(NC_DEPTH_BUCKETS - 1);  // 16+
+    return(uPlies);
+}
+
+static ULONG
+_NullCalibDeficitBucket(SCORE iDeficit)  // iEval - iBeta
+{
+    if (iDeficit < -300) return(0);
+    if (iDeficit < -100) return(1);
+    if (iDeficit < 0)    return(2);
+    if (iDeficit < 100)  return(3);
+    if (iDeficit < 300)  return(4);
+    return(5);
+}
+
+static ULONG
+_NullCalibMaterialBucket(ULONG uPiecesPerSide)  // generic: used for both MAX(both) and mover-only
+{
+    if (uPiecesPerSide <= 1) return(0);
+    if (uPiecesPerSide == 2) return(1);
+    if (uPiecesPerSide <= 5) return(2);
+    return(3);
+}
+
+static ULONG
+_NullCalibBucketIndex(ULONG uNullDepth,     // probe's own post-reduction depth (cost1 axis)
+                      ULONG uOuterDepth,    // real node's pre-reduction depth (cost2 axis)
+                      SCORE iDeficit,
+                      ULONG uMaxPiecesPerSide,    // MAX(white,black)
+                      ULONG uMoverPiecesPerSide)  // TotalNonPawnCount(pos, pos->uToMove)
+{
+    ULONG uNullDepthB = _NullCalibDepthBucket(uNullDepth);
+    ULONG uOuterDepthB = _NullCalibDepthBucket(uOuterDepth);
+    ULONG uDeficitB = _NullCalibDeficitBucket(iDeficit);
+    ULONG uMaxMaterialB = _NullCalibMaterialBucket(uMaxPiecesPerSide);
+    ULONG uMoverMaterialB = _NullCalibMaterialBucket(uMoverPiecesPerSide);
+    ULONG uIndex = uNullDepthB;
+    uIndex = uIndex * NC_DEPTH_BUCKETS + uOuterDepthB;
+    uIndex = uIndex * NC_DEFICIT_BUCKETS + uDeficitB;
+    uIndex = uIndex * NC_MATERIAL_BUCKETS + uMaxMaterialB;
+    uIndex = uIndex * NC_MATERIAL_BUCKETS + uMoverMaterialB;
+    ASSERT(uIndex < NC_TOTAL_BUCKETS);
+    return(uIndex);
+}
+#endif
+
 #ifdef DEBUG
 #define VERIFY_HASH_HIT                                \
     ASSERT(IS_VALID_SCORE(iScore));                    \
@@ -190,6 +315,15 @@ Search(IN SEARCHER_THREAD_CONTEXT *ctx,
     FLAG fAnyMoveEFPPruned = FALSE;
     FLAG fThisMoveEFPPruned = FALSE;
     SCORE iMoveSee;
+#ifdef TEST_NULL
+    FLAG fNullCalibEligible = FALSE;
+    FLAG fNullCalibProbeFH = FALSE;
+    ULONG uNullCalibBucket = 0;
+    UINT64 uNullCalibProbeNodeCost = 0;
+    UINT64 uNullCalibFirstMoveNodeCost = 0;
+    UINT64 uNullCalibRealLoopNodeCost = 0;
+    UINT64 uNullCalibNodesBeforeRealLoop = 0;
+#endif
 #ifdef DEBUG
     ASSERT(IS_VALID_SCORE(iAlpha));
     ASSERT(IS_VALID_SCORE(iBeta));
@@ -320,8 +454,18 @@ Search(IN SEARCHER_THREAD_CONTEXT *ctx,
     }
 
     // Probe interior node recognizers; allow probes of ondisk EGTB files
-    // if it looks like we can get hit.
-    switch(RecognLookup(ctx, &iScore, ctx->uPly <= (g_uIterateDepth / 2)))
+    // if we're shallow (near the root, where a first-touch mmap()/disk
+    // read is affordable) -- always probe there -- *or*, deeper, only
+    // if this exact position's lookup is likely resident (mincore()-
+    // gated for 6-man material; a handful of popcounts and one atomic
+    // load for 3-5 man -- see probe.c's EgtbProbeLikelyResident and
+    // CLAUDE.md). That residency check itself is deferred into
+    // RecognLookup (EGTB_PROBE_IF_RESIDENT), not computed here, so it's
+    // skipped entirely at any node a cheap interior recognizer already
+    // resolves -- see RecognLookup's own header comment.
+    switch(RecognLookup(ctx, &iScore,
+                         (ctx->uPly <= (g_uIterateDepth / 2)) ?
+                         EGTB_PROBE_ALWAYS : EGTB_PROBE_IF_RESIDENT))
     {
         case UNRECOGNIZED:
             break;
@@ -362,6 +506,83 @@ Search(IN SEARCHER_THREAD_CONTEXT *ctx,
     }
 
     GENERATE_NO_MOVES;
+
+#ifdef TEST_NULL
+    // See _NullCalibBucketIndex's comment above for the full design.
+    // Structural eligibility only -- deliberately does *not* consult
+    // fSkipNull/fThreat/WeShouldTryNullmovePruning's margin table or
+    // material threshold, since those are exactly the heuristics this
+    // harness exists to evaluate. Note: like any double-search
+    // diagnostic, this probe's recursive Search() call warms the hash
+    // and history/countermove tables along its own subtree before the
+    // real move loop below runs at this same node -- a known,
+    // unavoidable side effect of measuring this way, not a bug.
+    //
+    // MUST run after GENERATE_NO_MOVES, not before: that macro is what
+    // sets ctx->sMoveStack.uBegin[ctx->uPly+1] (from uEnd[ctx->uPly]),
+    // and recursing into ctx->uPly+1 before it runs uses a stale
+    // leftover index from whatever unrelated node last occupied that
+    // ply slot -- crashed hard (MVF_MOVE_SEARCHED-already-set asserts,
+    // corrupting a real, in-flight move a few frames up) the first time
+    // this was tried with the block placed above GENERATE_NO_MOVES.
+    fNullCalibEligible = ((FALSE == pf->fAvoidNullmove) &&
+                          (FALSE == pi->fInCheck) &&
+                          (iBeta != +INFINITY_SCORE) &&
+                          (NODE_PV != pi->eNodeType) &&
+                          (FALSE == fThreat) &&
+                          (FALSE == fSkipNull));
+    if (fNullCalibEligible)
+    {
+        ULONG uMaxPiecesPerSide = MAXU(TotalNonPawnCount(pos, WHITE),
+                                       TotalNonPawnCount(pos, BLACK));
+        ULONG uMoverPiecesPerSide = TotalNonPawnCount(pos, pos->uToMove);
+        SCORE iNullCalibScore;
+        UINT64 u64NodesBefore;
+
+        uNullCalibBucket = _NullCalibBucketIndex(uNextDepth,
+                                                 uDepth,
+                                                 (SCORE)(iEval - iBeta),
+                                                 uMaxPiecesPerSide,
+                                                 uMoverPiecesPerSide);
+        VERIFY(MakeMove(ctx, NULLMOVE));
+        pf->fAvoidNullmove = TRUE;
+        u64NodesBefore = ctx->sCounters.tree.u64TotalNodeCount;
+        iNullCalibScore = -Search(ctx, -iBeta, -iBeta + 1, uNextDepth);
+        uNullCalibProbeNodeCost = ctx->sCounters.tree.u64TotalNodeCount - u64NodesBefore;
+        pf->fAvoidNullmove = FALSE;
+        UnmakeMove(ctx, NULLMOVE);
+
+        if (!WE_SHOULD_STOP_SEARCHING)
+        {
+            // Direct increments, not INC() -- these calibration counters
+            // must not depend on PERF_COUNTERS also being defined; a
+            // TEST_NULL=1 build without PERF_COUNTERS=1 would otherwise
+            // silently collect nothing (INC() is a no-op without it).
+            fNullCalibProbeFH = (iNullCalibScore >= iBeta);
+            ctx->sCounters.tree.u64NCAttempts[uNullCalibBucket]++;
+            ctx->sCounters.tree.u64NCNullProbeNodeSum[uNullCalibBucket] +=
+                uNullCalibProbeNodeCost;
+            if (fNullCalibProbeFH)
+            {
+                ctx->sCounters.tree.u64NCNullFH[uNullCalibBucket]++;
+            }
+        }
+        else
+        {
+            fNullCalibEligible = FALSE;  // don't commit ground truth either
+        }
+        // Snapshot right here, before production's own null-move logic
+        // and the real move loop below -- the delta read at "end:" is
+        // the true total cost of however this node actually gets
+        // resolved from this point on (whether that's production's own
+        // separate null success, one real move, or several).
+        if (fNullCalibEligible)
+        {
+            uNullCalibNodesBeforeRealLoop = ctx->sCounters.tree.u64TotalNodeCount;
+        }
+    }
+#endif
+
     if (!fSkipNull &&
         !fThreat &&
         WeShouldTryNullmovePruning(ctx,
@@ -925,6 +1146,24 @@ Search(IN SEARCHER_THREAD_CONTEXT *ctx,
                 {
                     // First move, full a..b window.
                     ASSERT(uLegalMoves == 1);
+#ifdef TEST_NULL
+                    // Cost side of the null-move calibration harness'
+                    // try-vs-skip EV estimate: this is the "R=0"
+                    // alternative the null probe (see above) is being
+                    // weighed against -- the real first move's own
+                    // full-depth subtree cost. uLegalMoves==1 guarantees
+                    // this fires exactly once per node.
+                    if (fNullCalibEligible)
+                    {
+                        UINT64 u64NodesBeforeFirstMove =
+                            ctx->sCounters.tree.u64TotalNodeCount;
+                        iScore = -Search(ctx, -iBeta, -iAlpha, uNextDepth);
+                        uNullCalibFirstMoveNodeCost =
+                            ctx->sCounters.tree.u64TotalNodeCount -
+                            u64NodesBeforeFirstMove;
+                    }
+                    else
+#endif
                     iScore = -Search(ctx, -iBeta, -iAlpha, uNextDepth);
                 }
                 else
@@ -1162,6 +1401,58 @@ Search(IN SEARCHER_THREAD_CONTEXT *ctx,
         }
     }
 #endif
+#ifdef TEST_NULL
+    // Commit the calibration record stashed at the diagnostic probe
+    // site above, now that iBestScore's fate against iBeta (this
+    // node's real, ground-truth fail-high outcome) is known.
+    if (fNullCalibEligible &&
+        (!WE_SHOULD_STOP_SEARCHING) &&
+        IS_VALID_SCORE(iBestScore))
+    {
+        // Direct increments -- see the probe site's comment on why
+        // these can't use INC().
+        FLAG fRealFH = (iBestScore >= iBeta);
+        if (fRealFH)
+        {
+            ctx->sCounters.tree.u64NCRealFH[uNullCalibBucket]++;
+            if (fNullCalibProbeFH)
+            {
+                ctx->sCounters.tree.u64NCBothFH[uNullCalibBucket]++;
+            }
+        }
+        else if (fNullCalibProbeFH)
+        {
+            // False positive: the null-move probe predicted a fail-high
+            // that the real move loop didn't confirm -- exactly the
+            // "null-move lie" case that a low-material verification
+            // search exists to catch in production. Dump a capped
+            // number of example FENs per bucket (same "capped FEN
+            // sampler" idea as eval.c's MARGINSAFETY_EXCEEDED logging)
+            // so these can be inspected by hand for genuine zugzwang
+            // our TotalNonPawnCount/ValueOfMaterialInTroubleAfterNull
+            // safety net is missing, vs. ordinary search noise.
+            static ULONG uFalsePositiveFenSamples[NC_TOTAL_BUCKETS];
+            if (uFalsePositiveFenSamples[uNullCalibBucket] < 5)
+            {
+                CHAR *pszFen = PositionToFen(pos);
+                if (NULL != pszFen)
+                {
+                    uFalsePositiveFenSamples[uNullCalibBucket]++;
+                    Trace("NULLCALIB_FALSE_POSITIVE bucket=%lu depth=%lu "
+                          "fen=%s\n",
+                          uNullCalibBucket, uDepth, pszFen);
+                    SystemFreeMemory(pszFen);
+                }
+            }
+        }
+        ctx->sCounters.tree.u64NCFirstMoveNodeSum[uNullCalibBucket] +=
+            uNullCalibFirstMoveNodeCost;
+        uNullCalibRealLoopNodeCost = ctx->sCounters.tree.u64TotalNodeCount -
+                                     uNullCalibNodesBeforeRealLoop;
+        ctx->sCounters.tree.u64NCRealLoopNodeSum[uNullCalibBucket] +=
+            uNullCalibRealLoopNodeCost;
+    }
+#endif
     DTLeaveNode(ctx, FALSE, iBestScore, mvBest);
     return(iBestScore);
 }
@@ -1994,9 +2285,17 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx,
     }
     DTEnterNode(ctx, 0, TRUE, iAlpha, iBeta);
 
-    // Probe interior node recognizers; do not allow probes into ondisk
-    // EGTB files since we are in qsearch.
-    switch(RecognLookup(ctx, &iScore, FALSE))
+    // Probe interior node recognizers.  Qsearch runs at the deepest,
+    // most performance-sensitive part of the tree, so this used to
+    // unconditionally refuse EGTB probes -- but a qsearch node with few
+    // enough pieces to be in the tablebase at all is exactly the
+    // "trades have happened near a leaf" case the EGTB is most useful
+    // for: only probe when this position's lookup is likely resident
+    // (EGTB_PROBE_IF_RESIDENT -- RecognLookup itself decides, via
+    // EgtbProbeLikelyResident, only on the branch that's actually about
+    // to call ProbeEGTB, so a node an interior recognizer resolves
+    // never pays for the residency check at all) -- see CLAUDE.md.
+    switch(RecognLookup(ctx, &iScore, EGTB_PROBE_IF_RESIDENT))
     {
         case UNRECOGNIZED:
             break;