From 379a03bbd993247c8de9c1f1b163fcfab2fa1d69 Mon Sep 17 00:00:00 2001 From: Scott Gasch Date: Tue, 8 Sep 2026 18:47:22 -0700 Subject: Land super-lazy exit, material-based lazy floor, qsearch futility rework Brings in the last remaining piece from stash@{0}: the super-lazy exit point (material-only pre-check before the regular lazy gate), a material-bucket floor under the regular lazy exit's margin (iSwingFloorByArmy), and search.c's qsearch futility rework (FUTILITY_BASE_MARGIN_BY_SOURCE, indexed by which Eval() exit tier produced the score). Required Eval()'s signature change from a single SCORE* to SCORE(*)[2] (positional estimate per side instead of one munged magnitude) -- search.c's futility margin folds in rgiPositional[pos->uToMove], which the earlier bad-trades investigation found to be a meaningfully predictive signal. search.c and chess.h brought in wholesale from the stash (both were either completely untouched by prior commits or contained no divergence worth preserving). eval.c required hand-merging on top of this session's already-applied bad-trades fix, B-over-N removal, and xColor/reorder cleanups -- ported the super-lazy exit block, the regular-lazy material floor, the per-color piPositional writes (all three exit sites: super-lazy, regular-lazy x2, full-eval), the super-lazy calibration harness (RecordSuperLazyMarginSafetySwing, dual-regime DumpMarginSafetyCalibration), and moved uArmyScaler/ uNumTrapped initialization to match the new ordering the super-lazy exit depends on. Verified: clean release + DEBUG build (only the previously-flagged _EvalTrappedPieces warning), DEBUG smoke test pass, and a 40-game st1 match against clean 434fa04 (score 0.487, llr -0.04) -- landing this margin machinery as-is from the stash, before any retuning, does not regress strength on its own. This confirms the original regression (0.15-0.225 score seen early in this investigation) was fully explained by the bad-trades unsigned-underflow bug, not by these margins being unsound. Values are the original, as-derived-from-calibration ones (see inline comments: SUPER_LAZY_MARGIN_BY_ARMY from 100 positions/sd8 with ~15-20% headroom, iSwingFloorByArmy from 1500 positions/sd8 with ~25% headroom, FUTILITY_BASE_MARGIN_BY_SOURCE from a separate 1500-position/sd6 surprise-rate calibration). Not yet retuned -- suspected to carry more headroom than necessary, which costs real search speed (speed=depth). Next step: re-run the margin-safety calibration fresh against the current build and trim the super-lazy and regular-lazy floor headroom down from the built-in database, leaving the qsearch futility margins alone (different calibration method, already risk-tolerant by construction). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ka9o3S2eKqh4jNfmxVZ6fH --- src/chess.h | 161 ++++++++++++------ src/eval.c | 410 ++++++++++++++++++++++++++++++++++++++-------- src/search.c | 524 +++++++++++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 931 insertions(+), 164 deletions(-) diff --git a/src/chess.h b/src/chess.h index 702defb..78c0f94 100755 --- a/src/chess.h +++ b/src/chess.h @@ -941,19 +941,9 @@ typedef struct _COUNTERS UINT64 u64AvoidNullSuccess; UINT64 u64AvoidNullFailures; #endif - UINT64 u64EvalHashHits; UINT64 u64LazyEvals; - UINT64 u64FullEvals; - // Placeholder counters for root.c's per-tier eval-exit - // reporting -- eval.c's super-lazy exit itself hasn't been - // re-applied yet (see stash), so these always read 0 for now; - // that's accurate, not a stub bug, since no super-lazy exit - // exists in this build to increment them. UINT64 u64SuperLazyEvals; - UINT64 u64CyclesSuperLazyExit; - UINT64 u64CyclesLazyExit; - UINT64 u64CyclesFullEvalExit; - UINT64 u64CyclesEvalSuperLazy; + UINT64 u64FullEvals; UINT64 u64CyclesInEval; // @@ -1021,6 +1011,30 @@ typedef struct _COUNTERS UINT64 u64CyclesEvalLazyDecision; UINT64 u64CyclesEvalCountKingSafetyDefects; UINT64 u64CyclesEvalFileStormDefects; + + // + // 2026-09-07: the super-lazy exit (eval.c, before the regular + // LAZY_EVAL block) is a much cheaper material-only check than + // the regular lazy decision, so it gets its own cycle counter + // rather than folding into u64CyclesEvalLazyDecision -- a hit + // here never reaches the regular lazy-decision code at all. + // + UINT64 u64CyclesEvalSuperLazy; + + // Per-exit-path *total call cost*, i.e. the full entry-to-exit + // elapsed time for a call that left via that path -- not to be + // confused with the sub-timers above, which measure a single + // segment's cost regardless of how the call eventually exited. + // These three are mutually exclusive by construction (each + // Eval() call adds to exactly one) and sum to u64CyclesInEval, + // so u64CyclesInEval / (u64SuperLazyEvals + u64LazyEvals + + // u64FullEvals) is a real overall average, and each bucket + // divided by its own matching count (u64SuperLazyEvals, + // u64LazyEvals, u64FullEvals) gives a clean per-path average -- + // see root.c's eval-exit-breakdown report. + UINT64 u64CyclesSuperLazyExit; + UINT64 u64CyclesLazyExit; + UINT64 u64CyclesFullEvalExit; } tree; @@ -1240,6 +1254,24 @@ typedef struct _SEARCHER_THREAD_CONTEXT // real (uninstrumented) search, not every node Eval() is ever // called on. FLAG fCalibrateCandidate; +#endif + // Which Eval() exit tier supplied the piPositional value most + // recently returned to this thread -- set inside Eval() itself + // (all three exit points), read right after the Eval() call in + // QSearch. Was CALIBRATE_QSEARCH_FUTILITY-only; promoted to + // unconditional 2026-09-08 since QSearch's real futility margin + // now indexes FUTILITY_BASE_MARGIN_BY_SOURCE with it, not just + // the calibration harness. See EVAL_POSITIONAL_SOURCE_* below. + ULONG uLastPositionalSource; +#ifdef CALIBRATE_QSEARCH_FUTILITY + // TRUE for the duration of a diagnostic "what if we hadn't pruned + // this move" re-search -- search.c's qsearch futility gate checks + // this and forces iFutility wide open so the diagnostic subtree + // itself isn't contaminated by the same pruning being measured. + // Never set recursively: a rejection can't occur while this is + // already TRUE (iFutility being wide open means nothing gets + // rejected), so there's no risk of runaway nested diagnosis. + FLAG fDiagUnprunedSubtree; #endif CHAR szLastPV[SMALL_STRING_LEN_CHAR]; } @@ -2820,14 +2852,27 @@ IsDraw(SEARCHER_THREAD_CONTEXT *ctx); // search.c // #define QPLIES_OF_NON_CAPTURE_CHECKS (2) -#define FUTILITY_BASE_MARGIN (50) -// Compatibility aliases for root.c/main.c's per-tier reporting, which -// expects these three names -- search.c hasn't been split into -// per-tier margins yet (still one flat FUTILITY_BASE_MARGIN), so all -// three alias the same value until that split is re-applied. -#define FUTILITY_BASE_MARGIN_FULL FUTILITY_BASE_MARGIN -#define FUTILITY_BASE_MARGIN_LAZY FUTILITY_BASE_MARGIN -#define FUTILITY_BASE_MARGIN_SUPERLAZY FUTILITY_BASE_MARGIN + +// Which Eval() exit tier supplied a given piPositional value -- +// SEARCHER_THREAD_CONTEXT.uLastPositionalSource above. Was diagnostic- +// only (CALIBRATE_QSEARCH_FUTILITY); promoted to unconditional +// 2026-09-08 once FUTILITY_BASE_MARGIN_BY_SOURCE (search.c) made it +// load-bearing for real search behavior, not just measurement. +#define EVAL_POSITIONAL_SOURCE_FULL (0) +#define EVAL_POSITIONAL_SOURCE_LAZY (1) +#define EVAL_POSITIONAL_SOURCE_SUPERLAZY (2) +#define EVAL_POSITIONAL_SOURCE_COUNT (3) + +// Per-tier qsearch futility base margins -- see search.c's +// FUTILITY_BASE_MARGIN_BY_SOURCE for the full derivation. Named here +// (rather than only as array entries in search.c) so eval.c's +// piPositional floor -- which only ever applies on the full-eval +// exit -- can reference the matching tier's constant directly instead +// of a stale flat value. +#define FUTILITY_BASE_MARGIN_FULL (450) +#define FUTILITY_BASE_MARGIN_LAZY (225) +#define FUTILITY_BASE_MARGIN_SUPERLAZY (275) + // Measured: disabling this entirely (see lmr_testing/RESULTS.md) is a // clear net loss across ringers/confident_quick/hard_quick, so IID itself // is load-bearing. The "is the top move crappy" gate in search.c's DO_IID @@ -3144,7 +3189,7 @@ PawnHashLookup(SEARCHER_THREAD_CONTEXT *ctx); // #define LAZY_EVAL #define LAZE_EVAL_BASE_SCORE 10 -#define LAZY_EVAL_BASE_MARGIN (75) // cheap material-only lazy exit margin; +#define LAZY_EVAL_BASE_MARGIN (75) // cheap material-only lazy exit margin; // widened by EstimatePositionalScore // if this isn't enough on its own // board_representation/EVAL.md section 9 (2026-09-06): below this @@ -3162,13 +3207,6 @@ PawnHashLookup(SEARCHER_THREAD_CONTEXT *ctx); extern const int g_iAhead[2]; extern const int g_iBehind[2]; -// No-op placeholder: eval.c's ROOK_FULL_HALF_OPEN_BONUS static cache -// (the thing InitEval() is meant to (re)build, called at startup and -// after every DNA reload) hasn't been re-applied yet -- see stash. -// Nothing to initialize until that cache exists. -void -InitEval(void); - ULONG DNABufferSizeBytes(); @@ -3184,14 +3222,15 @@ ImportEvalDNA(char *p); FLAG ReadEvalDNA(char *szFilename); - +void +InitEval(); SCORE -Eval(SEARCHER_THREAD_CONTEXT *, SCORE, SCORE, SCORE *); +Eval(SEARCHER_THREAD_CONTEXT *, SCORE, SCORE, SCORE (*)[2]); FLAG _EvalPasserRacesAgainstLoneKings(POSITION *, - PAWN_HASH_ENTRY *); + PAWN_HASH_ENTRY *); ULONG CountKingSafetyDefects(POSITION *pos, @@ -3212,6 +3251,22 @@ void DumpMarginSafetyCalibration(void); #endif +#ifdef CALIBRATE_QSEARCH_FUTILITY +// Which futility gate in _ShouldWeConsiderThisMove rejected a move -- +// see search.c's _QFutRecordSample. The recapture gate (used to be +// QFUT_GATE_RECAPTURE) was retired 2026-09-08 along with the +100 +// recapture bonus itself -- see the "recapture-shaped" comment in +// _ShouldWeConsiderThisMove for why (fRecaptureShaped never checked +// mv.cTo == mvLast.cTo, so it wasn't testing real recaptures). +#define QFUT_GATE_GENERIC_CAPTURE (0) +#define QFUT_GATE_CHECK_ROOK (1) +#define QFUT_GATE_CHECK_BISHOP (2) +#define QFUT_GATE_COUNT (3) + +void +DumpQSearchFutilityCalibration(void); +#endif + // // testeval.c // @@ -3352,22 +3407,13 @@ ParallelCompareUlong(ULONG uComparand, void *pComparators); ULONG CDECL ParallelCompareVector(void *pComparand, void *pComparators); -void CDECL -GetAttacks(SEE_LIST *pList, - POSITION *pos, - COOR cSquare, - ULONG uSide); - -void CDECL -SlowGetAttacks(SEE_LIST *pList, - POSITION *pos, - COOR cSquare, - ULONG uSide); - -// board_representation/MIGRATION.md section 3: bbPieces/bbPawns-backed -// GetAttacks primitive, verified correct (20,000-position sweep) and -// faster than asm GetAttacks (0.53-0.89x cycles/call across -// opening/middlegame/endgame -- see section 3's benchmark writeup). +// board_representation/MIGRATION.md sections 3/6/7: bbPieces/bbPawns- +// backed GetAttacks primitive. Verified correct (20,000-position +// sweep) and faster than the old mailbox implementation (0.53-0.89x +// cycles/call across opening/middlegame/endgame). Now the only +// GetAttacks implementation -- the old asm/CROUTINES mailbox versions +// (SlowGetAttacks, asm GetAttacks) were retired 2026-09-06 once all of +// section 7's retirement criteria cleared. void CDECL _GetAttacksBB(SEE_LIST *pList, POSITION *pos, @@ -3386,10 +3432,9 @@ _WhoAttacksSquareBB(POSITION *pos, BITBOARD bbOccupied); // _GetAttacksBB is the only implementation as of 2026-09-06 (see -// MIGRATION.md section 6) -- verified correct and faster than the old -// asm/CROUTINES mailbox versions (SlowGetAttacks, asm GetAttacks), -// which have been retired. Every call site written against the name -// "GetAttacks" didn't need touching when the default changed. +// MIGRATION.md section 7) -- this macro exists only so call sites +// written against the name "GetAttacks" didn't need touching when the +// old asm/CROUTINES mailbox versions were retired. #define GetAttacks _GetAttacksBB #ifdef _X86_ @@ -3596,6 +3641,12 @@ COMMAND(BenchCommand); void TestDraw(void); +void +TestRecogn(void); + +void +TestRecognExhaustiveKNKP(void); + // // probe.c // @@ -3683,6 +3734,16 @@ RecognLookup(SEARCHER_THREAD_CONTEXT *ctx, SCORE *piScore, FLAG fProbeEGTB); +// +// Unregistered/experimental recognizers, exposed only so testrecogn.c +// can exhaustively validate a candidate before it's ever re-registered +// in InitializeInteriorNodeRecognizers. Not reachable from the normal +// RecognLookup dispatch path. +// +ULONG +_RecognizeKNKP(SEARCHER_THREAD_CONTEXT *ctx, + SCORE *piScore); + SCORE GetRoughEvalScore(IN SEARCHER_THREAD_CONTEXT *ctx, IN SCORE iAlpha, diff --git a/src/eval.c b/src/eval.c index bc5f6f4..867c487 100755 --- a/src/eval.c +++ b/src/eval.c @@ -2820,6 +2820,43 @@ static UINT64 g_uMarginSwingSum[BASE_MARGIN_MAT_BUCKETS]; // moved"), this directly answers "was any real exit actually unsound". static UINT64 g_uMarginExceeded[BASE_MARGIN_MAT_BUCKETS]; +// Same four counters, kept in a fully separate set for the super-lazy +// exit (eval.c's material-only check before the regular lazy gate). +// Its own SUPER_LAZY_MARGIN was never run through this harness when +// it was added -- a much cheaper, much-larger-required-gap check than +// the regular lazy exit, so mixing its swings into the buckets above +// would wash out both regimes' signal. +static UINT64 g_uSuperLazyMarginSwingMax[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uSuperLazyMarginSwingCount[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uSuperLazyMarginSwingSum[BASE_MARGIN_MAT_BUCKETS]; +static UINT64 g_uSuperLazyMarginExceeded[BASE_MARGIN_MAT_BUCKETS]; + +static void +_RecordMarginSafetySwingInto(IN OUT UINT64 *puMax, + IN OUT UINT64 *puCount, + IN OUT UINT64 *puSum, + IN OUT UINT64 *puExceeded, + IN POSITION *pos, + IN SCORE iSwing, + IN SCORE iActualMarginUsed) +{ + ULONG uMatBucket = _MaterialBucket(pos); + + if (iSwing > iActualMarginUsed) + { + puExceeded[uMatBucket]++; + } + + ASSERT(iSwing >= 0); + puCount[uMatBucket]++; + puSum[uMatBucket] += (UINT64)iSwing; + if ((UINT64)iSwing > puMax[uMatBucket]) + { + puMax[uMatBucket] = (UINT64)iSwing; + } +} + + static void RecordMarginSafetySwing(IN POSITION *pos, IN SCORE iSwing, @@ -2846,52 +2883,58 @@ Return value: **/ { - ULONG uMatBucket = _MaterialBucket(pos); - - if (iSwing > iActualMarginUsed) - { - g_uMarginExceeded[uMatBucket]++; - } - - ASSERT(iSwing >= 0); - g_uMarginSwingCount[uMatBucket]++; - g_uMarginSwingSum[uMatBucket] += (UINT64)iSwing; - if ((UINT64)iSwing > g_uMarginSwingMax[uMatBucket]) - { - g_uMarginSwingMax[uMatBucket] = (UINT64)iSwing; - } + _RecordMarginSafetySwingInto(g_uMarginSwingMax, g_uMarginSwingCount, + g_uMarginSwingSum, g_uMarginExceeded, + pos, iSwing, iActualMarginUsed); } -void -DumpMarginSafetyCalibration(void) +static void +RecordSuperLazyMarginSafetySwing(IN POSITION *pos, + IN SCORE iSwing, + IN SCORE iActualMarginUsed) /** Routine description: - Print, per material bucket, the max and average |real - lazy| swing - observed among nodes that actually took a lazy exit this run - (command.c's "calibrate marginsafety"). The max is the number that - tells you how large LAZY_EVAL_BASE_MARGIN would need to be, at that - material level, before an exit could have been unsound. + Same as RecordMarginSafetySwing, but for the super-lazy exit's own + separate bucket set -- see the comment on the g_uSuperLazyMargin* + arrays above for why these are kept apart. Parameters: - void + POSITION *pos + SCORE iSwing : >= 0 + SCORE iActualMarginUsed : SUPER_LAZY_MARGIN Return value: void **/ +{ + _RecordMarginSafetySwingInto(g_uSuperLazyMarginSwingMax, + g_uSuperLazyMarginSwingCount, + g_uSuperLazyMarginSwingSum, + g_uSuperLazyMarginExceeded, + pos, iSwing, iActualMarginUsed); +} + + +static void +_DumpMarginSafetyCalibrationInto(IN CHAR *szLabel, + IN UINT64 *puMax, + IN UINT64 *puCount, + IN UINT64 *puSum, + IN UINT64 *puExceeded) { ULONG m; - Trace("Margin safety -- max/avg |real - lazy| swing among exits taken, " - "by material bucket:\n"); + Trace("%s -- max/avg |real - lazy| swing among exits taken, " + "by material bucket:\n", szLabel); for (m = 0; m < BASE_MARGIN_MAT_BUCKETS; m++) { - if (0 == g_uMarginSwingCount[m]) + if (0 == puCount[m]) { Trace(" material bucket %u (scaler %u-%u): no exits taken\n", m, m * BASE_MARGIN_MAT_WIDTH, @@ -2904,11 +2947,48 @@ Return value: COMPILER_LONGLONG_UNSIGNED_FORMAT "\n", m, m * BASE_MARGIN_MAT_WIDTH, (m * BASE_MARGIN_MAT_WIDTH) + BASE_MARGIN_MAT_WIDTH - 1, - g_uMarginSwingMax[m], - (double)g_uMarginSwingSum[m] / (double)g_uMarginSwingCount[m], - g_uMarginSwingCount[m], g_uMarginExceeded[m]); + puMax[m], (double)puSum[m] / (double)puCount[m], + puCount[m], puExceeded[m]); } } + + +void +DumpMarginSafetyCalibration(void) +/** + +Routine description: + + Print, per material bucket, the max and average |real - lazy| swing + observed among nodes that actually took a lazy exit this run + (command.c's "calibrate marginsafety") -- once for the regular lazy + exit (LAZY_EVAL_BASE_MARGIN / EstimatePositionalScore) and once for + the super-lazy exit (SUPER_LAZY_MARGIN), which is a different-enough + regime (cheaper check, much larger required gap) that lumping the + two together would hide whichever one is actually unsound. Either + section's max is the number that answers "how large would that + margin need to be, at that material level, before an exit there + could have been unsound." + +Parameters: + + void + +Return value: + + void + +**/ +{ + _DumpMarginSafetyCalibrationInto("Regular lazy margin safety", + g_uMarginSwingMax, g_uMarginSwingCount, + g_uMarginSwingSum, g_uMarginExceeded); + _DumpMarginSafetyCalibrationInto("Super lazy margin safety", + g_uSuperLazyMarginSwingMax, + g_uSuperLazyMarginSwingCount, + g_uSuperLazyMarginSwingSum, + g_uSuperLazyMarginExceeded); +} #endif // CALIBRATE_MARGIN_SAFETY @@ -3036,6 +3116,36 @@ Return value: *piAlphaMargin += iKingTerm + iResidualP90; *piBetaMargin += iKingTerm + iResidualP90; + + // 2026-09-08: the p90 estimate above, while within its own + // documented "~10% wrong" design tolerance, still let the regular + // lazy exit's real swing exceed the margin actually used 2.11- + // 2.38% of the time at combined army scaler 16-31 (CALIBRATE_ + // MARGIN_SAFETY, 1500 real-game positions, tests/twic_sample.ep_, + // sd 8) -- not because the swing itself is much bigger there (max + // observed 855/779, comparable to every other bucket's 603-701), + // but because REDUCED_MATERIAL_DOWN_SCALER shrinks the king term + // faster than the swing actually shrinks at that material level. + // Floor the combined margin at the measured max-per-bucket (with + // ~25% headroom) rather than re-deriving the whole p90 shape -- + // this leaves the estimate above as the primary driver wherever + // it's already wide enough (buckets 4-7, where it already clears + // these floors) and only kicks in where it wasn't. Buckets 0-1 + // never take a regular lazy exit at all (see LAZY_EVAL_MIN_ + // MATERIAL) so were never measured; use bucket 2's floor there + // defensively rather than leaving them unfloored. Provisional -- + // re-derive with `calibrate marginsafety` if this margin's shape + // changes again. + { + static const SCORE iSwingFloorByArmy[8] = + { + 1069, 1069, 1069, 974, 821, 876, 796, 754, + }; + ULONG uMatBucket = MINU( + 7, (pos->uArmyScaler[WHITE] + pos->uArmyScaler[BLACK]) / 8); + *piAlphaMargin = MAX(*piAlphaMargin, iSwingFloorByArmy[uMatBucket]); + *piBetaMargin = MAX(*piBetaMargin, iSwingFloorByArmy[uMatBucket]); + } } @@ -5033,7 +5143,7 @@ SCORE Eval(IN SEARCHER_THREAD_CONTEXT *ctx, IN SCORE iAlpha, IN SCORE iBeta, - OUT SCORE *piPositional) + OUT SCORE (*piPositional)[2]) /** Routine description: @@ -5043,11 +5153,16 @@ Parameters: SEARCHER_THREAD_CONTEXT *ctx, SCORE iAlpha, SCORE iBeta, - SCORE *piPositional : if non-NULL, filled in with a magnitude - (always >= 0) estimating the non-material component of the - score -- exact if a full eval ran, a cheap estimate otherwise. - Callers must treat it as an estimate either way; it's only - ever used to size a pruning margin, never as a hard fact. + SCORE (*piPositional)[2] : if non-NULL, filled in with an estimate + of the non-material component of the score for each side, + indexed by absolute color (WHITE/BLACK) -- exact if a full + eval ran, a cheap estimate otherwise. On a lazy exit this is + always >= 0 (an optimistic margin); on a full eval it can be + negative (a side whose positional terms net worse than its + raw material), floored at -FUTILITY_BASE_MARGIN_FULL/2 so it can + only shrink a caller's margin so far. Callers must treat it + as an estimate either way; it's only ever used to size a + pruning margin, never as a hard fact. Return value: @@ -5061,8 +5176,8 @@ Return value: PAWN_HASH_ENTRY *pHash; COOR c; ULONG u; - ULONG uColor; - ULONG xColor; + ULONG uColor = pos->uToMove; + ULONG xColor = FLIP(uColor); BITBOARD bb; FLAG fDeferred; #ifdef EVAL_TIME @@ -5079,6 +5194,12 @@ Return value: FLAG fWouldHaveExited = FALSE; SCORE iSavedLazyScore = 0; SCORE iSavedLazyPositional = 0; + // Same idea, kept separate from the pair above: the super-lazy + // exit (added 2026-09-07) is a much cheaper, much-larger-gap check + // than the regular lazy exit and was never hooked into this + // harness -- see RecordSuperLazyMarginSafetySwing below. + FLAG fWouldHaveExitedSuperLazy = FALSE; + SCORE iSavedSuperLazyScore = 0; #endif ASSERT(IS_VALID_SCORE(iAlpha)); ASSERT(IS_VALID_SCORE(iBeta)); @@ -5086,17 +5207,140 @@ Return value: ASSERT((pos->iMaterialBalance[WHITE] * -1) == pos->iMaterialBalance[BLACK]); - pos->uNumTrapped[BLACK] = pos->uNumTrapped[WHITE] = 0; - - pos->iScore[BLACK] = - (pos->uPawnMaterial[BLACK] + pos->uNonPawnMaterial[BLACK]); - pos->iScore[WHITE] = - (pos->uPawnMaterial[WHITE] + pos->uNonPawnMaterial[WHITE]); + pos->iScore[uColor] = + (pos->uPawnMaterial[uColor] + pos->uNonPawnMaterial[uColor]); + pos->iScore[xColor] = + (pos->uPawnMaterial[xColor] + pos->uNonPawnMaterial[xColor]); #ifdef EVAL_DUMP EvalTraceClear(); Trace("Material:\n%d\t\t%d\n", pos->iScore[WHITE], pos->iScore[BLACK]); #endif + // Initialize army scalers here (moved ahead of the super-lazy exit + // point below, 2026-09-08 -- their only inputs, uNonPawnMaterial, + // are already available this early, and the super-lazy margin + // table needs a material bucket before it can run). We skip lazy + // eval if we're too late into an endgame. + pos->uArmyScaler[BLACK] = pos->uNonPawnMaterial[BLACK] - VALUE_KING; + pos->uArmyScaler[WHITE] = pos->uNonPawnMaterial[WHITE] - VALUE_KING; + pos->uArmyScaler[BLACK] /= VALUE_PAWN; + pos->uArmyScaler[WHITE] /= VALUE_PAWN; + ASSERT(!(pos->uArmyScaler[BLACK] & 0x80000000)); + ASSERT(!(pos->uArmyScaler[WHITE] & 0x80000000)); + pos->uArmyScaler[BLACK] = MINU(31, pos->uArmyScaler[BLACK]); + pos->uArmyScaler[WHITE] = MINU(31, pos->uArmyScaler[WHITE]); + ASSERT(pos->uArmyScaler[BLACK] >= 0); + ASSERT(pos->uArmyScaler[BLACK] <= 31); + ASSERT(pos->uArmyScaler[WHITE] >= 0); + ASSERT(pos->uArmyScaler[WHITE] <= 31); + + // Super-lazy exit point. +#ifdef LAZY_EVAL + // 2026-09-08: was a single flat 625 for every material level. + // CALIBRATE_MARGIN_SAFETY data (100 real-game positions, sd 8, + // tests/twic_sample.ep_) showed that's badly unsound at low + // material -- up to 9.3% of super-lazy exits in bare-king-plus-a- + // little-material positions (combined army scaler 0-7) had a real + // swing exceeding 625, max observed 1710 -- while richer positions + // (combined scaler 32+) never came close (max 893, well under + // 625... actually under 900, still comfortably bounded). Table + // indexed the same way _MaterialBucket buckets the calibration + // data (combined army scaler / 8, 8 buckets), so it can be + // re-derived directly from a "calibrate marginsafety" run. Values + // here are the observed max per bucket rounded up with headroom + // (~15-20%), not a hard theoretical bound -- provisional pending a + // larger/deeper calibration run; re-check before trusting this at + // sd well beyond 8. + static const SCORE SUPER_LAZY_MARGIN_BY_ARMY[8] = + { + 2000, 1800, 1800, 1750, 1000, 850, 850, 850, + }; + ULONG uSuperLazyMatBucket = MINU( + 7, (pos->uArmyScaler[WHITE] + pos->uArmyScaler[BLACK]) / 8); + SCORE iSuperLazyMargin = SUPER_LAZY_MARGIN_BY_ARMY[uSuperLazyMatBucket]; + { +#ifdef EVAL_TIME + UINT64 uSuperLazyTimer = SystemReadTimeStampCounter(); +#endif + iScoreForSideToMove = (pos->iScore[uColor] -pos->iScore[xColor]); + ASSERT(IS_VALID_SCORE(iScoreForSideToMove)); + if (iScoreForSideToMove + iSuperLazyMargin < iAlpha) + { + ASSERT(iScoreForSideToMove < iAlpha); + INC(ctx->sCounters.tree.u64SuperLazyEvals); + if (NULL != piPositional) + { + // No positional credit here -- a super-lazy verdict is + // already a confident material-only read that we're + // well outside the window; handing back extra slack + // just re-inflates the qsearch futility margin and + // defeats the point of taking the cheap exit. Let + // FUTILITY_BASE_MARGIN_BY_SOURCE[SUPERLAZY] alone + // govern from here. + (*piPositional)[WHITE] = 0; + (*piPositional)[BLACK] = 0; + ctx->uLastPositionalSource = EVAL_POSITIONAL_SOURCE_SUPERLAZY; + } +#ifdef EVAL_TIME + ctx->sCounters.tree.u64CyclesEvalSuperLazy += + (SystemReadTimeStampCounter() - uSuperLazyTimer); +#endif +#ifdef CALIBRATE_MARGIN_SAFETY + // Don't exit yet -- let the real full eval run below so we + // can measure the true swing against SUPER_LAZY_MARGIN, + // then restore this exact (zeroed) piPositional right + // before `end:` so the caller sees what a normal build + // would have returned. + fWouldHaveExitedSuperLazy = TRUE; + iSavedSuperLazyScore = iScoreForSideToMove; +#else +#ifdef EVAL_TIME + ctx->sCounters.tree.u64CyclesEvalPreLazy += + (SystemReadTimeStampCounter() - uTimer); + ctx->sCounters.tree.u64CyclesSuperLazyExit += + (SystemReadTimeStampCounter() - uTimer); + ctx->sCounters.tree.u64CyclesInEval += + (SystemReadTimeStampCounter() - uTimer); +#endif + goto end; +#endif + } + if (iScoreForSideToMove - iSuperLazyMargin > iBeta) + { + ASSERT(iScoreForSideToMove > iBeta); + INC(ctx->sCounters.tree.u64SuperLazyEvals); + if (NULL != piPositional) + { + (*piPositional)[WHITE] = 0; + (*piPositional)[BLACK] = 0; + ctx->uLastPositionalSource = EVAL_POSITIONAL_SOURCE_SUPERLAZY; + } +#ifdef EVAL_TIME + ctx->sCounters.tree.u64CyclesEvalSuperLazy += + (SystemReadTimeStampCounter() - uSuperLazyTimer); +#endif +#ifdef CALIBRATE_MARGIN_SAFETY + fWouldHaveExitedSuperLazy = TRUE; + iSavedSuperLazyScore = iScoreForSideToMove; +#else +#ifdef EVAL_TIME + ctx->sCounters.tree.u64CyclesEvalPreLazy += + (SystemReadTimeStampCounter() - uTimer); + ctx->sCounters.tree.u64CyclesSuperLazyExit += + (SystemReadTimeStampCounter() - uTimer); + ctx->sCounters.tree.u64CyclesInEval += + (SystemReadTimeStampCounter() - uTimer); +#endif + goto end; +#endif + } +#ifdef EVAL_TIME + ctx->sCounters.tree.u64CyclesEvalSuperLazy += + (SystemReadTimeStampCounter() - uSuperLazyTimer); +#endif + } +#endif + // // Pawn eval. Note: if fDeferred comes back as TRUE then we have // neither cleared nor initialized the attack tables. This is @@ -5273,7 +5517,9 @@ Return value: INC(ctx->sCounters.tree.u64LazyEvals); if (NULL != piPositional) { - *piPositional = iAlphaMargin; + (*piPositional)[WHITE] = iAlphaMargin; + (*piPositional)[BLACK] = iAlphaMargin; + ctx->uLastPositionalSource = EVAL_POSITIONAL_SOURCE_LAZY; } #ifdef EVAL_TIME ctx->sCounters.tree.u64CyclesEvalPreLazy += @@ -5300,7 +5546,9 @@ Return value: INC(ctx->sCounters.tree.u64LazyEvals); if (NULL != piPositional) { - *piPositional = iBetaMargin; + (*piPositional)[WHITE] = iBetaMargin; + (*piPositional)[BLACK] = iBetaMargin; + ctx->uLastPositionalSource = EVAL_POSITIONAL_SOURCE_LAZY; } #ifdef EVAL_TIME ctx->sCounters.tree.u64CyclesEvalPreLazy += @@ -5375,20 +5623,7 @@ Return value: // // Pre-compute some common terms used in per-piece evals: // - // This is a scaler based on the size of the army for each side. - // - pos->uArmyScaler[BLACK] = pos->uNonPawnMaterial[BLACK] - VALUE_KING; - pos->uArmyScaler[WHITE] = pos->uNonPawnMaterial[WHITE] - VALUE_KING; - pos->uArmyScaler[BLACK] /= VALUE_PAWN; - pos->uArmyScaler[WHITE] /= VALUE_PAWN; - ASSERT(!(pos->uArmyScaler[BLACK] & 0x80000000)); - ASSERT(!(pos->uArmyScaler[WHITE] & 0x80000000)); - pos->uArmyScaler[BLACK] = MINU(31, pos->uArmyScaler[BLACK]); - pos->uArmyScaler[WHITE] = MINU(31, pos->uArmyScaler[WHITE]); - ASSERT(pos->uArmyScaler[BLACK] >= 0); - ASSERT(pos->uArmyScaler[BLACK] <= 31); - ASSERT(pos->uArmyScaler[WHITE] >= 0); - ASSERT(pos->uArmyScaler[WHITE] <= 31); + pos->uNumTrapped[BLACK] = pos->uNumTrapped[WHITE] = 0; pos->iReducedMaterialDownScaler[BLACK] = pos->iReducedMaterialDownScaler[WHITE] = 0; @@ -5695,18 +5930,61 @@ Return value: (iScoreForSideToMove * (SCORE)uDrawDist / 16); } - // - // Adjust dynamic positional component. - // - iAlphaMargin = abs(pos->iMaterialBalance[pos->uToMove] - iScoreForSideToMove); + // Adjust dynamic positional component. Unlike the lazy-exit + // margins above (always >= 0 by construction), this can go + // negative -- a side whose positional terms (king safety, pawn + // structure, hanging pieces, ...) net worse than its raw material + // gets a genuinely negative value here. search.c's futility + // margin (FUTILITY_BASE_MARGIN_BY_SOURCE[side's tier] + + // piPositional[side]) deliberately lets that shrink the margin -- + // a positionally-bad side is less likely to be saved by an + // unexamined quiet move -- but floor it so one bad eval term + // can't collapse the margin arbitrarily far; this only ever runs + // on the full-eval exit, so -FUTILITY_BASE_MARGIN_FULL/2 (not the + // lazy/super-lazy tiers) caps the damage at half of that tier's + // base. if (NULL != piPositional) { - *piPositional = iAlphaMargin; + (*piPositional)[WHITE] = + MAX(pos->iScore[WHITE] - + (SCORE)(pos->uPawnMaterial[WHITE] + pos->uNonPawnMaterial[WHITE]), + -(FUTILITY_BASE_MARGIN_FULL / 2)); + (*piPositional)[BLACK] = + MAX(pos->iScore[BLACK] - + (SCORE)(pos->uPawnMaterial[BLACK] + pos->uNonPawnMaterial[BLACK]), + -(FUTILITY_BASE_MARGIN_FULL / 2)); + ctx->uLastPositionalSource = EVAL_POSITIONAL_SOURCE_FULL; } g_Options.iLastEvalScore = iScoreForSideToMove; #ifdef CALIBRATE_MARGIN_SAFETY - if (TRUE == fWouldHaveExited) + if (TRUE == fWouldHaveExitedSuperLazy) + { + // + // Real (non-calibration) code checks super-lazy first and + // returns immediately on a hit -- so if this call would *also* + // have satisfied the regular lazy condition below, super-lazy + // is still what a normal build actually returns. Record/ + // restore this one, not the regular-lazy save in the other + // branch. + // + RecordSuperLazyMarginSafetySwing( + pos, abs(iScoreForSideToMove - iSavedSuperLazyScore), + iSuperLazyMargin); + iScoreForSideToMove = iSavedSuperLazyScore; + if (NULL != piPositional) + { + (*piPositional)[WHITE] = 0; + (*piPositional)[BLACK] = 0; + // CALIBRATE_MARGIN_SAFETY builds: the FULL tag set above + // reflects the fallthrough that just ran for measurement + // purposes, not what a normal build actually returns here + // -- fix it back up to match the restored (super-lazy) + // values. + ctx->uLastPositionalSource = EVAL_POSITIONAL_SOURCE_SUPERLAZY; + } + } + else if (TRUE == fWouldHaveExited) { // // The real swing: how far the true, full-eval score actually @@ -5725,7 +6003,9 @@ Return value: iScoreForSideToMove = iSavedLazyScore; if (NULL != piPositional) { - *piPositional = iSavedLazyPositional; + (*piPositional)[WHITE] = iSavedLazyPositional; + (*piPositional)[BLACK] = iSavedLazyPositional; + ctx->uLastPositionalSource = EVAL_POSITIONAL_SOURCE_LAZY; } } #endif diff --git a/src/search.c b/src/search.c index 25318ed..fdcb955 100755 --- a/src/search.c +++ b/src/search.c @@ -1087,18 +1087,311 @@ Return value: FALSE if it can be skipped **/ -#define QSEARCH_CONSIDER_MARGIN (120) +#ifdef CALIBRATE_QSEARCH_FUTILITY +// 2026-09-08: measures whether _ShouldWeConsiderThisMove's futility +// gates are set correctly, by -- at the moment a move would be +// rejected -- actually searching it anyway (fully unpruned, via +// ctx->fDiagUnprunedSubtree) and checking whether it would genuinely +// have raised alpha. Never changes real search behavior: the +// diagnostic re-search's result is used only to log a sample, then +// discarded (same non-interference pattern as CALIBRATE_MARGIN_SAFETY +// in eval.c). +// +// Bucketed by (gate, margin-neutral distance short of the relevant +// threshold, piPositional bucket, which Eval() tier supplied +// piPositional) so a single run answers several questions at once: +// is the margin itself wide enough (distance-vs-surprise-rate curve +// within a gate), does piPositional actually predict surprises +// (compare curves across piPositional buckets at the same distance), +// and does that answer differ by tier (super-lazy vs regular-lazy vs +// full eval). +#define QFUT_DIST_BUCKETS (6) +#define QFUT_POS_BUCKETS (4) + +static UINT64 g_uQFutTries[QFUT_GATE_COUNT][QFUT_DIST_BUCKETS][QFUT_POS_BUCKETS][EVAL_POSITIONAL_SOURCE_COUNT]; +static UINT64 g_uQFutSurprises[QFUT_GATE_COUNT][QFUT_DIST_BUCKETS][QFUT_POS_BUCKETS][EVAL_POSITIONAL_SOURCE_COUNT]; + +static ULONG +_QFutDistanceBucket(IN SCORE iDistance) +/* iDistance: how far short of the relevant threshold this move was + (positive = short; a move that actually cleared the bar never gets + here, so this should always be > 0, but negative/zero is folded + into bucket 0 defensively rather than asserting -- a measurement + harness should never crash a calibration run over its own bucketing + edge case). */ +{ + if (iDistance <= 25) return(0); + if (iDistance <= 50) return(1); + if (iDistance <= 100) return(2); + if (iDistance <= 200) return(3); + if (iDistance <= 400) return(4); + return(5); +} + +static ULONG +_QFutPositionalBucket(IN SCORE iPositional) +{ + if (iPositional < 0) return(0); + if (iPositional < 25) return(1); + if (iPositional < 75) return(2); + return(3); +} + +static void +_QFutDiagnoseReject(IN SEARCHER_THREAD_CONTEXT *ctx, + IN ULONG uMoveNum, + IN SCORE iAlpha, + IN SCORE iBeta, + IN SCORE iPositional, + IN ULONG uPositionalSource, + IN ULONG uGate, + IN SCORE iDistance, + IN FLAG fGeneratedChecks) +/** + +Routine description: + + A move is about to be rejected by one of _ShouldWeConsiderThisMove's + futility gates. Before rejecting it for real, search it anyway + (fully unpruned) to see whether it would actually have raised + alpha, and log the outcome. The diagnostic search's result is + discarded -- this function never changes what the caller does. + +Return value: + + void + +**/ +{ + MOVE mv; + SCORE iScore; + ULONG uDistBucket, uPosBucket; + + // Never diagnose from inside an already-diagnostic (fully + // unpruned) subtree -- structurally shouldn't happen anyway, since + // nothing gets rejected while fDiagUnprunedSubtree is set, but + // guard explicitly rather than relying on that. + if (TRUE == ctx->fDiagUnprunedSubtree) + { + return; + } + ASSERT(uGate < QFUT_GATE_COUNT); + ASSERT(uPositionalSource < EVAL_POSITIONAL_SOURCE_COUNT); + + mv = ctx->sMoveStack.mvf[uMoveNum].mv; + // Mirror QSearch's own move loop exactly: GenerateMoves only tags + // MVF_CHECK/the checking-move bit when checks were actually being + // generated this ply. Skipping this (as an earlier version of this + // function did) leaves the flag unset on a move that objectively + // does give check, which MakeMove/ply-info bookkeeping trusts + // blindly -- the next ply's fInCheck-vs-InCheck() consistency + // ASSERT (searchsup.c) catches the mismatch immediately. + if (FALSE == fGeneratedChecks) + { + mv.bvFlags |= WouldGiveCheck(ctx, mv); + } + if (FALSE == MakeMove(ctx, mv)) + { + return; + } + ctx->fDiagUnprunedSubtree = TRUE; + ctx->sSearchFlags.uQsearchDepth++; + iScore = -QSearch(ctx, -iBeta, -iAlpha); + ctx->sSearchFlags.uQsearchDepth--; + ctx->fDiagUnprunedSubtree = FALSE; + UnmakeMove(ctx, mv); + + uDistBucket = _QFutDistanceBucket(iDistance); + uPosBucket = _QFutPositionalBucket(iPositional); + g_uQFutTries[uGate][uDistBucket][uPosBucket][uPositionalSource]++; + if (iScore > iAlpha) + { + g_uQFutSurprises[uGate][uDistBucket][uPosBucket][uPositionalSource]++; + } +} + + +static CHAR *g_szQFutGateNames[QFUT_GATE_COUNT] = +{ + "generic capture/promo", + "checking capture/promo (VALUE_ROOK)", + "quiet check (VALUE_BISHOP)", +}; +static CHAR *g_szQFutSourceNames[EVAL_POSITIONAL_SOURCE_COUNT] = +{ + "full eval", + "regular lazy", + "super lazy", +}; + +void +DumpQSearchFutilityCalibration(void) +/** + +Routine description: + + Print, per (gate, piPositional-source tier), the surprise rate + (fraction of diagnostically-re-searched rejects that actually + raised alpha) by distance-short-of-threshold bucket, and + separately by piPositional bucket at the widest distance bucket -- + read the former for "is this gate's margin wide enough," the + latter (compared across piPositional buckets at a fixed distance) + for "does piPositional actually predict surprises." + +Parameters: + + void + +Return value: + + void + +**/ +{ + ULONG g, d, p, s; + static CHAR *szDistLabel[QFUT_DIST_BUCKETS] = + { "0-25", "25-50", "50-100", "100-200", "200-400", "400+" }; + static CHAR *szPosLabel[QFUT_POS_BUCKETS] = + { "<0", "0-25", "25-75", "75+" }; + + for (g = 0; g < QFUT_GATE_COUNT; g++) + { + Trace("QSearch futility gate: %s\n", g_szQFutGateNames[g]); + for (s = 0; s < EVAL_POSITIONAL_SOURCE_COUNT; s++) + { + UINT64 u64TotalTries = 0; + for (d = 0; d < QFUT_DIST_BUCKETS; d++) + { + for (p = 0; p < QFUT_POS_BUCKETS; p++) + { + u64TotalTries += g_uQFutTries[g][d][p][s]; + } + } + if (0 == u64TotalTries) + { + continue; + } + Trace(" source=%s (n=%" COMPILER_LONGLONG_UNSIGNED_FORMAT "):\n", + g_szQFutSourceNames[s], u64TotalTries); + Trace(" by distance short of bar (summed over piPositional buckets):\n"); + for (d = 0; d < QFUT_DIST_BUCKETS; d++) + { + UINT64 u64Tries = 0, u64Surprises = 0; + for (p = 0; p < QFUT_POS_BUCKETS; p++) + { + u64Tries += g_uQFutTries[g][d][p][s]; + u64Surprises += g_uQFutSurprises[g][d][p][s]; + } + if (0 == u64Tries) + { + continue; + } + Trace(" dist %8s: %6.2f%% surprise rate (n=%" + COMPILER_LONGLONG_UNSIGNED_FORMAT ")\n", + szDistLabel[d], + 100.0 * (double)u64Surprises / (double)u64Tries, + u64Tries); + } + Trace(" by piPositional bucket (summed over distance buckets):\n"); + for (p = 0; p < QFUT_POS_BUCKETS; p++) + { + UINT64 u64Tries = 0, u64Surprises = 0; + for (d = 0; d < QFUT_DIST_BUCKETS; d++) + { + u64Tries += g_uQFutTries[g][d][p][s]; + u64Surprises += g_uQFutSurprises[g][d][p][s]; + } + if (0 == u64Tries) + { + continue; + } + Trace(" pos %6s: %6.2f%% surprise rate (n=%" + COMPILER_LONGLONG_UNSIGNED_FORMAT ")\n", + szPosLabel[p], + 100.0 * (double)u64Surprises / (double)u64Tries, + u64Tries); + } + // The marginals above can each look flat on their own even + // when piPositional genuinely matters (its effect might + // only show up at a fixed distance) -- this is the actual + // Q2 answer: read a single distance row across columns. If + // the percentages don't move across piPositional buckets + // at a fixed distance, it isn't predictive; if they fall + // as piPositional rises, it is. + Trace(" cross-tab, surprise%% (rows=distance, cols=piPositional " + "%s/%s/%s/%s):\n", + szPosLabel[0], szPosLabel[1], szPosLabel[2], szPosLabel[3]); + for (d = 0; d < QFUT_DIST_BUCKETS; d++) + { + UINT64 u64RowTries = 0; + for (p = 0; p < QFUT_POS_BUCKETS; p++) + { + u64RowTries += g_uQFutTries[g][d][p][s]; + } + if (0 == u64RowTries) + { + continue; + } + Trace(" dist %8s:", szDistLabel[d]); + for (p = 0; p < QFUT_POS_BUCKETS; p++) + { + UINT64 u64Tries = g_uQFutTries[g][d][p][s]; + if (0 == u64Tries) + { + Trace(" n/a "); + } + else + { + Trace(" %5.1f%% (n=%5" COMPILER_LONGLONG_UNSIGNED_FORMAT ")", + 100.0 * (double)g_uQFutSurprises[g][d][p][s] / + (double)u64Tries, + u64Tries); + } + } + Trace("\n"); + } + } + } +} +#endif // CALIBRATE_QSEARCH_FUTILITY + + +// Flat bonus added to a checking capture/promotion's own (real, +// winning/even) iMoveValue before comparing against iFutility -- +// replaces the retired VALUE_ROOK position-level cutoff for exactly +// this population (see _ShouldWeConsiderThisMove's comment at its use +// site for why). Provisional -- no calibration data yet for this +// specific split; re-measure with `calibrate qsearchfutility`. +#define CHECK_BONUS (150) + static FLAG INLINE _ShouldWeConsiderThisMove(IN SEARCHER_THREAD_CONTEXT *ctx, IN ULONG uMoveNum, IN SCORE iFutility, - IN FLAG fGeneratedChecks) + IN FLAG fGeneratedChecks +#ifdef CALIBRATE_QSEARCH_FUTILITY + , IN SCORE iAlpha + , IN SCORE iBeta + , IN SCORE iPositional + , IN ULONG uPositionalSource +#endif + ) { +#ifdef DEBUG MOVE mvLast = ctx->sPlyInfo[ctx->uPly - 1].mv; +#endif MOVE mv = ctx->sMoveStack.mvf[uMoveNum].mv; ULONG uColor; - SCORE i; + SCORE iMoveValue; + // TRUE once iMoveValue holds a real, comparable SEE-derived value + // (winning/even captures and promotions) rather than the raw, + // not-directly-comparable generation-time score a losing capture + // keeps. Used both for real control flow (the checking-move + // branch below needs to know which of two populations it's + // looking at) and, under CALIBRATE_QSEARCH_FUTILITY, to decide + // whether a reject is worth diagnosing. + FLAG fHaveMoveValue = FALSE; ASSERT(!IS_CHECKING_MOVE(mvLast)); ASSERT(!InCheck(&(ctx->sPosition), ctx->sPosition.uToMove)); @@ -1119,12 +1412,13 @@ _ShouldWeConsiderThisMove(IN SEARCHER_THREAD_CONTEXT *ctx, } } - i = ctx->sMoveStack.mvf[uMoveNum].iValue; - if (i >= SORT_THESE_FIRST) + iMoveValue = ctx->sMoveStack.mvf[uMoveNum].iValue; + if (iMoveValue >= SORT_THESE_FIRST) { - i &= STRIP_OFF_FLAGS; - ASSERT(i >= 0); - i -= MOVE_SCORE_ORDERING_BIAS(mv); + fHaveMoveValue = TRUE; + iMoveValue &= STRIP_OFF_FLAGS; + ASSERT(iMoveValue >= 0); + iMoveValue -= MOVE_SCORE_ORDERING_BIAS(mv); if (mv.pCaptured) { // If there are very few pieces left on the board, @@ -1151,21 +1445,30 @@ _ShouldWeConsiderThisMove(IN SEARCHER_THREAD_CONTEXT *ctx, } // Don't trust the SEE alone for alpha pruning decisions. - i = MAXU(i, PIECE_VALUE(mv.pCaptured)); - - // Also try hard not to prune recaps, the bad trade - // penalty can make them look "futile" sometimes. - if ((PIECE_VALUE(mv.pCaptured) == - PIECE_VALUE(mvLast.pCaptured)) && - (i + 200 + QSEARCH_CONSIDER_MARGIN > iFutility)) - { - return(TRUE); - } + iMoveValue = MAXU(iMoveValue, PIECE_VALUE(mv.pCaptured)); + + // RETIRED 2026-09-08: used to give recaptures (same + // captured-piece value as mvLast) a flat +100 bonus + // here ("the bad trade penalty can make them look + // futile sometimes"). CALIBRATE_QSEARCH_FUTILITY data + // showed it wasn't testing real recaptures at all -- + // this check never compared mv.cTo to mvLast.cTo, so + // "recapture-shaped" meant "captured a same-valued + // piece anywhere on the board," diluting genuine + // recaptures (usually safe) with unrelated captures + // (not specially safe) under one bonus. That mismatch + // is the more likely explanation for the gate's high, + // slowly-decaying surprise rate (13.25% at distance + // 0-25, still 3.29% at 400+) than the constant being + // merely too small. Removed rather than re-tuned; + // reintroduce with a same-square check if a bonus + // still looks warranted once the generic gate's own + // margin is fixed. } // Otherwise, even if a move is even/winning, make sure it // brings the score up to at least somewhere near alpha. - if (i + QSEARCH_CONSIDER_MARGIN > iFutility) + if (iMoveValue > iFutility) { return(TRUE); } @@ -1175,10 +1478,73 @@ _ShouldWeConsiderThisMove(IN SEARCHER_THREAD_CONTEXT *ctx, // that checked or a "futile" winning capture/prom that may or // may not check. Be more willing to play checking captures // even if they look bad. + // + // RETIRED VALUE_ROOK 2026-09-08: used to judge both + // populations below by a single position-level "iFutility < + // VALUE_ROOK" cutoff, discarding iMoveValue entirely even + // when a real one existed. CALIBRATE_QSEARCH_FUTILITY data + // showed a flat, non-decaying-with-distance surprise rate + // (8-16%, no better far past the threshold than right at it) + // -- the signature of a position-level test standing in for a + // move-level question it can't actually answer. Split into + // the two populations that were being conflated: a move with + // a real (winning/even) iMoveValue gets the same value-plus- + // flat-bonus treatment as any other capture; a move with no + // usable value (SEE already called it losing) falls back to + // GetCheckSee's real tactical judgment, exactly like the + // quiet-check (VALUE_BISHOP) branch already does below. if (IS_CHECKING_MOVE(mv) && (TRUE == fGeneratedChecks)) { - return(iFutility < +VALUE_ROOK); + if (TRUE == fHaveMoveValue) + { + FLAG fConsider = (iMoveValue + CHECK_BONUS > iFutility); +#ifdef CALIBRATE_QSEARCH_FUTILITY + if (FALSE == fConsider) + { + _QFutDiagnoseReject(ctx, uMoveNum, iAlpha, iBeta, + iPositional, uPositionalSource, + QFUT_GATE_CHECK_ROOK, + iFutility - (iMoveValue + CHECK_BONUS), + fGeneratedChecks); + } +#endif + return(fConsider); + } + { + FLAG fConsider = (GetCheckSee(ctx, mv, uMoveNum) >= 0); +#ifdef CALIBRATE_QSEARCH_FUTILITY + if (FALSE == fConsider) + { + // No value-level threshold left for this + // population (that's the point) -- iFutility + // itself is the only position-level number left + // to bucket by, used as-is rather than a + // difference from some retired constant. + _QFutDiagnoseReject(ctx, uMoveNum, iAlpha, iBeta, + iPositional, uPositionalSource, + QFUT_GATE_CHECK_ROOK, + iFutility, + fGeneratedChecks); + } +#endif + return(fConsider); + } } + +#ifdef CALIBRATE_QSEARCH_FUTILITY + // Falls through to the final reject below -- either not a + // checking move, or a checking move we weren't generating + // checks for this ply (rare; the generic gate is still what + // decided this, so tag it the same way). + if (TRUE == fHaveMoveValue) + { + _QFutDiagnoseReject(ctx, uMoveNum, iAlpha, iBeta, + iPositional, uPositionalSource, + QFUT_GATE_GENERIC_CAPTURE, + iFutility - iMoveValue, + fGeneratedChecks); + } +#endif } else { @@ -1194,7 +1560,20 @@ _ShouldWeConsiderThisMove(IN SEARCHER_THREAD_CONTEXT *ctx, { return(TRUE); } - return(GetCheckSee(ctx, mv, uMoveNum) >= 0); + { + FLAG fConsider = (GetCheckSee(ctx, mv, uMoveNum) >= 0); +#ifdef CALIBRATE_QSEARCH_FUTILITY + if (FALSE == fConsider) + { + _QFutDiagnoseReject(ctx, uMoveNum, iAlpha, iBeta, + iPositional, uPositionalSource, + QFUT_GATE_CHECK_BISHOP, + iFutility - (SCORE)VALUE_BISHOP, + fGeneratedChecks); + } +#endif + return(fConsider); + } } return(FALSE); } @@ -1249,21 +1628,6 @@ QSearchFromCheckNoStandPat(IN SEARCHER_THREAD_CONTEXT *ctx, if ((pf->uQsearchDepth < pf->uQsearchCheckDepth) && (pf->uQsearchDepth < g_uIterateDepth / 4) && (pf->fCouldStandPat[pos->uToMove] == FALSE) && - // - // Threshold doubled 2026-09-06 (board_representation/ - // EVAL.md section 9): was ">2", tuned against the old - // CHECK_VECTOR-based CountKingSafetyDefects. The bitboard - // rewrite (real blocker-aware slider attacks, queen 2x - // weighting) runs systematically hotter for the same - // underlying danger, so the un-rescaled old threshold was - // firing far more liberally than intended -- observed - // directly as a runaway check-extension cascade (85x+ - // branching for a single ply) on a real position. Not yet - // independently recalibrated against real data the way - // iKingSwingP90 was; doubling is a stopgap matching the - // rough inflation this counter picked up, pending a real - // measurement. - // (CountKingSafetyDefects(pos, pos->uToMove) > 4)) { if ((uMoveCount == 1) || @@ -1344,6 +1708,27 @@ QSearchFromCheckNoStandPat(IN SEARCHER_THREAD_CONTEXT *ctx, } +// 2026-09-08: was one flat FUTILITY_BASE_MARGIN (150) regardless of +// which Eval() exit tier produced iEval/rgiPositional this call. +// CALIBRATE_QSEARCH_FUTILITY data (1500 real-game positions, +// tests/twic_sample.ep_, sd 6) showed the three tiers need very +// different margins to reach a similar surprise rate: full-eval +// source was still failing 1.65-6.4% of diagnosed rejects even +// hundreds of centipawns short of the bar, regular-lazy was already +// close to safe (1.26% down to 0.43%), super-lazy was only risky very +// close to the bar (7.96% at distance 0-25, 0.54% by 400+). Indexed +// by ctx->uLastPositionalSource (set inside Eval() -- see +// EVAL_POSITIONAL_SOURCE_* in chess.h). Provisional, derived from one +// run at sd 6; re-derive with `calibrate qsearchfutility` if search +// behavior affecting typical qsearch iEval/iFutility gaps changes. +static const SCORE FUTILITY_BASE_MARGIN_BY_SOURCE[EVAL_POSITIONAL_SOURCE_COUNT] = +{ + FUTILITY_BASE_MARGIN_FULL, // EVAL_POSITIONAL_SOURCE_FULL + FUTILITY_BASE_MARGIN_LAZY, // EVAL_POSITIONAL_SOURCE_LAZY + FUTILITY_BASE_MARGIN_SUPERLAZY, // EVAL_POSITIONAL_SOURCE_SUPERLAZY +}; + + /** Routine description: @@ -1380,7 +1765,7 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx, SCORE iScore; SCORE iEval; SCORE iFutility; - SCORE iPositional; + SCORE rgiPositional[2]; ULONG x; #ifdef PERF_COUNTERS ULONG uLegalMoves; @@ -1459,7 +1844,7 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx, } ASSERT(!InCheck(pos, pos->uToMove)); - iEval = iBestScore = Eval(ctx, iAlpha, iBeta, &iPositional); + iEval = iBestScore = Eval(ctx, iAlpha, iBeta, &rgiPositional); // If that Eval (above) was full (i.e. not lazy) it may have set // en prise and trapped piece indicators. Likewise, other nodes @@ -1480,6 +1865,7 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx, } else { + ctx->sSearchFlags.fCouldStandPat[pos->uToMove] = TRUE; if (iBestScore > iAlpha) { iAlpha = iBestScore; @@ -1490,7 +1876,6 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx, goto end; } } - ctx->sSearchFlags.fCouldStandPat[pos->uToMove] = TRUE; } // He did not choose to stand pat here or we did not allow it. We @@ -1504,20 +1889,51 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx, iFutility = 0; if (iAlpha < +NMATE) { - iFutility = iAlpha - (FUTILITY_BASE_MARGIN + iPositional) - iEval; +#ifdef CALIBRATE_QSEARCH_FUTILITY + if (TRUE == ctx->fDiagUnprunedSubtree) + { + // We're inside a diagnostic "what if this rejected move + // had been searched anyway" re-search (see + // _QFutDiagnoseReject) -- force every gate in this + // function wide open so the diagnostic subtree isn't + // contaminated by the same pruning it exists to measure. + iFutility = -NMATE; + } + else +#endif +#ifdef DIAG_NO_QSEARCH_FUTILITY + // Diagnostic-only (never defined in a normal build): forces + // _ShouldWeConsiderThisMove's gates wide open so every + // capture/checking-move margin question in this file passes + // trivially, to measure the node-count/time cost of qsearch + // futility pruning as a whole -- not for shipping, just for + // sizing how expensive a "fully unpruned diagnostic subtree" + // would be for the qsearch-futility calibration harness. + iFutility = -NMATE; +#else + ASSERT(ctx->uLastPositionalSource < EVAL_POSITIONAL_SOURCE_COUNT); + iFutility = iAlpha - + (FUTILITY_BASE_MARGIN_BY_SOURCE[ctx->uLastPositionalSource] + + rgiPositional[pos->uToMove]) - iEval; iFutility = MAX0(iFutility); +#endif } // We know we are not in check. If we are early in the qsearch, // and the other side has not yet been able to stand pat yet, and // we have material OR we have hanging pieces, generate checks // here too. Checks are a "good way" to escape from "trouble". - fIncludeChecks = ((pf->uQsearchDepth < pf->uQsearchCheckDepth) && - (((pf->fCouldStandPat[FLIP(pos->uToMove)] == FALSE) && - (pos->uNonPawnMaterial[pos->uToMove] > - (VALUE_KING + VALUE_BISHOP))) || - (FALSE == ctx->sSearchFlags.fCouldStandPat[pos->uToMove]))); - + fIncludeChecks = ( + (pf->uQsearchDepth < pf->uQsearchCheckDepth) && + ( + ( + (pf->fCouldStandPat[FLIP(pos->uToMove)] == FALSE) && + (pos->uNonPawnMaterial[pos->uToMove] > (VALUE_KING + VALUE_BISHOP)) + ) + || + (FALSE == ctx->sSearchFlags.fCouldStandPat[pos->uToMove]) + ) + ); GenerateMoves(ctx, NULLMOVE, _WhatToGen[fIncludeChecks]); #ifdef PERF_COUNTERS @@ -1546,7 +1962,14 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx, if (FALSE == _ShouldWeConsiderThisMove(ctx, x, iFutility, - fIncludeChecks)) + fIncludeChecks +#ifdef CALIBRATE_QSEARCH_FUTILITY + , iAlpha + , iBeta + , rgiPositional[pos->uToMove] + , ctx->uLastPositionalSource +#endif + )) { continue; } @@ -1610,9 +2033,12 @@ QSearch(IN SEARCHER_THREAD_CONTEXT *ctx, // Readjust futility margin here; it can be wider now. if (iAlpha < +NMATE) { + ASSERT(ctx->uLastPositionalSource < + EVAL_POSITIONAL_SOURCE_COUNT); iFutility = (iAlpha - - (FUTILITY_BASE_MARGIN + - iPositional) - + (FUTILITY_BASE_MARGIN_BY_SOURCE[ + ctx->uLastPositionalSource] + + rgiPositional[pos->uToMove]) - iEval); iFutility = MAX0(iFutility); } -- cgit v1.3