diff options
| author | Scott Gasch <[email protected]> | 2026-10-20 09:26:44 -0700 |
|---|---|---|
| committer | Scott Gasch <[email protected]> | 2026-10-20 09:26:44 -0700 |
| commit | c8c6bbc2c1a103e5b07b079a0f8bdf66c7724d80 (patch) | |
| tree | b6c2137c9ac577fa83b1e948fa0311d53dfe445d /src/nnueeval.c | |
| parent | 3b5d3677bb14ae14728a1c4b221176bb83667d92 (diff) | |
Wire NNUE eval into search, build the training pipeline, and add a fast
move/eval-agreement benchmark against Stockfish (tests/lop)
Implements the architecture decisions in src/docs/NNUE.md: HalfKAv2_hm
feature set, int16/int8 quantization, per-ply accumulator backup for
king moves, and a lazy-eval-ready integration into search.c/eval.c.
Adds the full annotate_pgns/ training pipeline (extraction, K-fit/
labeling, quiet filtering, tensor building, training, export) plus an
end-to-end orchestrator (run_nnue_pipeline.py) and validation tooling
(sanity_check_net.py, compare_vs_stockfish_net.py). Adds tests/lop:
a 10k-position list-of-positions benchmark comparing engine eval rank
and move choice against pre-computed Stockfish/lc0 references, much
cheaper than a full self-play match for day-to-day progress checks.
First sd10-matched self-play result against classic HCE: LLR +2.28
(H1=+2.94), -72=119+169 over 360 games, confirming the net is
"smarter per node" at matched depth. A real-time (--st 4) match is
currently weaker (candidate loses ~3 plies of depth to a ~3.3-3.7x
nps gap against an unrecalibrated search), consistent with
NNUE.md's adoption-criterion section -- pruning/reduction constants
still need retuning for the new eval's score scale before a fair
real-time comparison is possible.
Co-Authored-By: Claude Sonnet 5 <[email protected]>
Diffstat (limited to 'src/nnueeval.c')
| -rw-r--r-- | src/nnueeval.c | 406 |
1 files changed, 377 insertions, 29 deletions
diff --git a/src/nnueeval.c b/src/nnueeval.c index 6ee57d5..7caadef 100644 --- a/src/nnueeval.c +++ b/src/nnueeval.c @@ -28,13 +28,23 @@ Author: **/ #include "chess.h" +#include <float.h> -float g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE]; +// g_rgfNnueWeight/g_rgfNnueBias (SHORT, quantized) and g_fNnueFtScale +// are defined in nnueaccum.c. The FC head below is quantized to CHAR +// (int8) per docs/NNUE.md's "Quantization plan" step 2 -- biases stay +// float32 (tiny, added post-dequant). g_rgfNnueActScale/WeightScale +// are read from the export file's header; defaults of 1.0f here only +// matter before any real weights are loaded (the all-zero placeholder +// table makes every activation 0 regardless of scale). +CHAR g_rgfNnueFc1Weight[NNUE_L2][2 * NNUE_ACCUMULATOR_SIZE]; float g_rgfNnueFc1Bias[NNUE_L2]; -float g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2]; +CHAR g_rgfNnueFc2Weight[NNUE_L3][NNUE_L2]; float g_rgfNnueFc2Bias[NNUE_L3]; -float g_rgfNnueFc3Weight[NNUE_L3]; +CHAR g_rgfNnueFc3Weight[NNUE_L3]; float g_rgfNnueFc3Bias; +float g_rgfNnueActScale[3] = { 1.0f, 1.0f, 1.0f }; +float g_rgfNnueWeightScale[3] = { 1.0f, 1.0f, 1.0f }; FLAG g_fNnueWeightsLoaded = FALSE; FLAG g_fUseNnueEval = FALSE; @@ -76,10 +86,12 @@ Return value: } if ((1 != fread(szMagic, 4, 1, fp)) || - (0 != memcmp(szMagic, "NNU1", 4))) + (0 != memcmp(szMagic, "NNU3", 4))) { Trace("NnueLoadWeights: %s is not a valid NNUE weights file " - "(bad magic)\n", szFilename); + "(bad magic -- expected \"NNU3\"; an older \"NNU1\"/" + "\"NNU2\" file predates the FC head's int8 quantization " + "and is no longer readable)\n", szFilename); fclose(fp); return FALSE; } @@ -104,9 +116,24 @@ Return value: return FALSE; } -#define READ_OR_FAIL(ptr, count) \ + if (1 != fread(&g_fNnueFtScale, sizeof(g_fNnueFtScale), 1, fp)) + { + Trace("NnueLoadWeights: %s truncated (ft_scale)\n", szFilename); + fclose(fp); + return FALSE; + } + if ((1 != fread(g_rgfNnueActScale, sizeof(g_rgfNnueActScale), 1, fp)) || + (1 != fread(g_rgfNnueWeightScale, sizeof(g_rgfNnueWeightScale), 1, fp))) + { + Trace("NnueLoadWeights: %s truncated (act/weight scales)\n", + szFilename); + fclose(fp); + return FALSE; + } + +#define READ_OR_FAIL(ptr, elsize, count) \ do { \ - if ((size_t)(count) != fread((ptr), sizeof(float), (count), fp)) \ + if ((size_t)(count) != fread((ptr), (elsize), (count), fp)) \ { \ Trace("NnueLoadWeights: %s truncated\n", szFilename); \ fclose(fp); \ @@ -114,14 +141,16 @@ Return value: } \ } while (0) - READ_OR_FAIL(g_rgfNnueWeight, (size_t)NNUE_DIMENSIONS * NNUE_ACCUMULATOR_SIZE); - READ_OR_FAIL(g_rgfNnueBias, NNUE_ACCUMULATOR_SIZE); - READ_OR_FAIL(g_rgfNnueFc1Weight, (size_t)NNUE_L2 * 2 * NNUE_ACCUMULATOR_SIZE); - READ_OR_FAIL(g_rgfNnueFc1Bias, NNUE_L2); - READ_OR_FAIL(g_rgfNnueFc2Weight, (size_t)NNUE_L3 * NNUE_L2); - READ_OR_FAIL(g_rgfNnueFc2Bias, NNUE_L3); - READ_OR_FAIL(g_rgfNnueFc3Weight, NNUE_L3); - READ_OR_FAIL(&g_rgfNnueFc3Bias, 1); + READ_OR_FAIL(g_rgfNnueWeight, sizeof(SHORT), + (size_t)NNUE_DIMENSIONS * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueBias, sizeof(SHORT), NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueFc1Weight, sizeof(CHAR), + (size_t)NNUE_L2 * 2 * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueFc1Bias, sizeof(float), NNUE_L2); + READ_OR_FAIL(g_rgfNnueFc2Weight, sizeof(CHAR), (size_t)NNUE_L3 * NNUE_L2); + READ_OR_FAIL(g_rgfNnueFc2Bias, sizeof(float), NNUE_L3); + READ_OR_FAIL(g_rgfNnueFc3Weight, sizeof(CHAR), NNUE_L3); + READ_OR_FAIL(&g_rgfNnueFc3Bias, sizeof(float), 1); #undef READ_OR_FAIL @@ -163,48 +192,367 @@ Return value: POSITION *pos = &ctx->sPosition; ULONG uStm = pos->uToMove; ULONG uOther = FLIP(uStm); - float rgfConcat[2 * NNUE_ACCUMULATOR_SIZE]; + UCHAR rgbAct1[2 * NNUE_ACCUMULATOR_SIZE]; // quantized fc1 input + UCHAR rgbAct2[NNUE_L2]; // quantized fc2 input + UCHAR rgbAct3[NNUE_L3]; // quantized fc3 input float rgfL2[NNUE_L2]; float rgfL3[NNUE_L3]; float fOut; ULONG i, j; + // Stage 0: dequantize the INT accumulator (scaled by + // g_fNnueFtScale, see chess.h's NNUE_ACCUMULATOR comment) back to + // float, apply ReLU, then requantize to UCHAR at AS1 + // (g_rgfNnueActScale[0]) -- this is "fc1's input" in the Python + // reference. Own-perspective-first concatenation matches the + // side-to-move-relative output convention. for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) { - float v = ctx->sNnueAccum[uStm].rgfValue[i]; - rgfConcat[i] = (v > 0.0f) ? v : 0.0f; + float v = (float)ctx->sNnueAccum[uStm].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + rgbAct1[i] = (UCHAR)MAX(0, MIN(255, (int)(v * g_rgfNnueActScale[0] + 0.5f))); } for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) { - float v = ctx->sNnueAccum[uOther].rgfValue[i]; - rgfConcat[NNUE_ACCUMULATOR_SIZE + i] = (v > 0.0f) ? v : 0.0f; + float v = (float)ctx->sNnueAccum[uOther].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + rgbAct1[NNUE_ACCUMULATOR_SIZE + i] = + (UCHAR)MAX(0, MIN(255, (int)(v * g_rgfNnueActScale[0] + 0.5f))); } + // Stage 1: fc1, int8 dot product (UCHAR activation x CHAR weight) + // accumulated in INT (32-bit) -- up to 512 terms, each at most + // 255*127, nowhere near INT overflow. Dequantize by dividing out + // both scales, add the (unquantized) bias, ReLU, requantize at + // AS2 for fc2's input. for (j = 0; j < NNUE_L2; j++) { - float s = g_rgfNnueFc1Bias[j]; + INT iSum = 0; for (i = 0; i < 2 * NNUE_ACCUMULATOR_SIZE; i++) { - s += g_rgfNnueFc1Weight[j][i] * rgfConcat[i]; + iSum += (INT)rgbAct1[i] * (INT)g_rgfNnueFc1Weight[j][i]; } - rgfL2[j] = (s > 0.0f) ? s : 0.0f; + float s = ((float)iSum / (g_rgfNnueActScale[0] * g_rgfNnueWeightScale[0])) + + g_rgfNnueFc1Bias[j]; + s = (s > 0.0f) ? s : 0.0f; + rgfL2[j] = s; + rgbAct2[j] = (UCHAR)MAX(0, MIN(255, (int)(s * g_rgfNnueActScale[1] + 0.5f))); } + // Stage 2: fc2, same pattern. for (j = 0; j < NNUE_L3; j++) { - float s = g_rgfNnueFc2Bias[j]; + INT iSum = 0; for (i = 0; i < NNUE_L2; i++) { - s += g_rgfNnueFc2Weight[j][i] * rgfL2[i]; + iSum += (INT)rgbAct2[i] * (INT)g_rgfNnueFc2Weight[j][i]; } - rgfL3[j] = (s > 0.0f) ? s : 0.0f; + float s = ((float)iSum / (g_rgfNnueActScale[1] * g_rgfNnueWeightScale[1])) + + g_rgfNnueFc2Bias[j]; + s = (s > 0.0f) ? s : 0.0f; + rgfL3[j] = s; + rgbAct3[j] = (UCHAR)MAX(0, MIN(255, (int)(s * g_rgfNnueActScale[2] + 0.5f))); } - fOut = g_rgfNnueFc3Bias; - for (i = 0; i < NNUE_L3; i++) + // Stage 3: fc3, single output, no ReLU (final scalar score). { - fOut += g_rgfNnueFc3Weight[i] * rgfL3[i]; + INT iSum = 0; + for (i = 0; i < NNUE_L3; i++) + { + iSum += (INT)rgbAct3[i] * (INT)g_rgfNnueFc3Weight[i]; + } + fOut = ((float)iSum / (g_rgfNnueActScale[2] * g_rgfNnueWeightScale[2])) + + g_rgfNnueFc3Bias; } return (SCORE)(fOut * 100.0f); } + + +void +NnueDumpActivationStats(SEARCHER_THREAD_CONTEXT *ctx) +/** + +Routine description: + + Diagnostic-only: re-runs NnueForward's same math but reports, per + layer, what fraction of quantized activations are clamped to the + [0,255] UCHAR rail (saturated) vs sitting strictly inside it, plus + the pre-clamp float stats. Built in response to the "loses a whole + queen and the eval barely moves" bug report, 2026-10-17 -- the + working theory is that ordinary middlegame positions saturate most + activations to the same rail regardless of material, so a single + piece's worth of accumulator change never survives the clamp to + reach the dense layers; only an extreme (near-decisive) material + swing moves enough units across the rail to register. This + function exists to confirm or refute that theory, not to fix + anything -- reference/measurement-only, not wired into search. + +Parameters: + + SEARCHER_THREAD_CONTEXT *ctx + +Return value: + + void + +**/ +{ + POSITION *pos = &ctx->sPosition; + ULONG uStm = pos->uToMove; + ULONG uOther = FLIP(uStm); + UCHAR rgbAct1[2 * NNUE_ACCUMULATOR_SIZE]; + UCHAR rgbAct2[NNUE_L2]; + UCHAR rgbAct3[NNUE_L3]; + ULONG i, j; + ULONG uClampLow, uClampHigh; + float fMin, fMax, fSum; + + // Stage 0 (fc1 input) -- same math as NnueForward, but track the + // pre-clamp float range and how many of the 512 entries hit each + // rail. + uClampLow = uClampHigh = 0; + fMin = FLT_MAX; + fMax = -FLT_MAX; + fSum = 0.0f; + for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) + { + float v = (float)ctx->sNnueAccum[uStm].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + float q = v * g_rgfNnueActScale[0] + 0.5f; + if (q <= 0.0f) uClampLow++; + if (q >= 255.0f) uClampHigh++; + if (q < fMin) fMin = q; + if (q > fMax) fMax = q; + fSum += q; + rgbAct1[i] = (UCHAR)MAX(0, MIN(255, (int)q)); + } + for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) + { + float v = (float)ctx->sNnueAccum[uOther].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + float q = v * g_rgfNnueActScale[0] + 0.5f; + if (q <= 0.0f) uClampLow++; + if (q >= 255.0f) uClampHigh++; + if (q < fMin) fMin = q; + if (q > fMax) fMax = q; + fSum += q; + rgbAct1[NNUE_ACCUMULATOR_SIZE + i] = (UCHAR)MAX(0, MIN(255, (int)q)); + } + Trace("fc1 input (%u units): %u at 0, %u at 255, range [%.1f, %.1f], mean %.1f\n", + 2 * NNUE_ACCUMULATOR_SIZE, uClampLow, uClampHigh, fMin, fMax, + fSum / (float)(2 * NNUE_ACCUMULATOR_SIZE)); + + // Stage 1 (fc2 input) + uClampLow = uClampHigh = 0; + fMin = FLT_MAX; + fMax = -FLT_MAX; + fSum = 0.0f; + for (j = 0; j < NNUE_L2; j++) + { + INT iSum = 0; + for (i = 0; i < 2 * NNUE_ACCUMULATOR_SIZE; i++) + { + iSum += (INT)rgbAct1[i] * (INT)g_rgfNnueFc1Weight[j][i]; + } + float s = ((float)iSum / (g_rgfNnueActScale[0] * g_rgfNnueWeightScale[0])) + + g_rgfNnueFc1Bias[j]; + s = (s > 0.0f) ? s : 0.0f; + float q = s * g_rgfNnueActScale[1] + 0.5f; + if (q <= 0.0f) uClampLow++; + if (q >= 255.0f) uClampHigh++; + if (q < fMin) fMin = q; + if (q > fMax) fMax = q; + fSum += q; + rgbAct2[j] = (UCHAR)MAX(0, MIN(255, (int)q)); + } + Trace("fc2 input (%u units): %u at 0, %u at 255, range [%.1f, %.1f], mean %.1f\n", + (ULONG)NNUE_L2, uClampLow, uClampHigh, fMin, fMax, + fSum / (float)NNUE_L2); + + // Stage 2 (fc3 input) + uClampLow = uClampHigh = 0; + fMin = FLT_MAX; + fMax = -FLT_MAX; + fSum = 0.0f; + for (j = 0; j < NNUE_L3; j++) + { + INT iSum = 0; + for (i = 0; i < NNUE_L2; i++) + { + iSum += (INT)rgbAct2[i] * (INT)g_rgfNnueFc2Weight[j][i]; + } + float s = ((float)iSum / (g_rgfNnueActScale[1] * g_rgfNnueWeightScale[1])) + + g_rgfNnueFc2Bias[j]; + s = (s > 0.0f) ? s : 0.0f; + float q = s * g_rgfNnueActScale[2] + 0.5f; + if (q <= 0.0f) uClampLow++; + if (q >= 255.0f) uClampHigh++; + if (q < fMin) fMin = q; + if (q > fMax) fMax = q; + fSum += q; + rgbAct3[j] = (UCHAR)MAX(0, MIN(255, (int)q)); + } + Trace("fc3 input (%u units): %u at 0, %u at 255, range [%.1f, %.1f], mean %.1f\n", + (ULONG)NNUE_L3, uClampLow, uClampHigh, fMin, fMax, + fSum / (float)NNUE_L3); +} + + +// Calibration-only lazy-eval probe (docs/NNUE.md's "Lazy-evaluation +// tiering plan"). A linear regression fit offline over fc1's own +// input (512-wide: the ReLU'd, concatenated accumulator halves) -- +// NOT a second trained net, just a cheap linear read of a +// representation the real net already produces. Completely separate +// weight file/format from NnueLoadWeights' production .nnue files; +// this is measurement-only infrastructure, consulted only under +// CALIBRATE_MARGIN_SAFETY (see eval.c's Eval()) and never used to +// skip any real computation yet. +float g_rgfNnueProbeMu[2 * NNUE_ACCUMULATOR_SIZE]; +float g_rgfNnueProbeSigma[2 * NNUE_ACCUMULATOR_SIZE]; +float g_rgfNnueProbeWeight[2 * NNUE_ACCUMULATOR_SIZE]; +float g_fNnueProbeBias; +FLAG g_fNnueProbeLoaded = FALSE; +FLAG g_fUseNnueLazyProbe = FALSE; + +// Runtime overrides for experimenting with how loose the NNUE lazy/ +// super-lazy margins can go without rebuilding -- 2026-10-16, see +// docs/NNUE.md "paradox: safety doesn't win games" discussion. +// g_iNnueLazyMarginOverride < 0 means "use eval.c's calibrated +// per-material-bucket table as before"; >= 0 means "use this flat cp +// value for every bucket instead". g_fUseNnueSuperLazy/ +// g_iNnueSuperLazyMargin gate a brand new, even cheaper pre-filter +// (pure material balance, no net at all) ahead of the probe -- this +// was previously measurement-only (see eval.c's CALIBRATE_MARGIN_SAFETY +// block) and is now wired into the real Eval() path for the first time. +INT g_iNnueLazyMarginOverride = -1; +FLAG g_fUseNnueSuperLazy = FALSE; +SCORE g_iNnueSuperLazyMargin = 0; + + +FLAG +NnueProbeLoadWeights(CHAR *szFilename) +/** + +Routine description: + + Loads the lazy-eval probe's weights -- a flat file written by + annotate_pgns/export_nnue_weights.py's sibling fitting script: a + 4-byte magic ("NPRB"), a uint32 dimension (must equal + 2*NNUE_ACCUMULATOR_SIZE), then mu/sigma/weight (each that many + float32s) and a single float32 bias, in that order. + +Parameters: + + CHAR *szFilename + +Return value: + + FLAG : TRUE on success + +**/ +{ + FILE *fp; + CHAR szMagic[4]; + ULONG uDim; + + g_fNnueProbeLoaded = FALSE; + fp = fopen(szFilename, "rb"); + if (NULL == fp) + { + Trace("NnueProbeLoadWeights: could not open %s\n", szFilename); + return FALSE; + } + + if ((1 != fread(szMagic, 4, 1, fp)) || + (0 != memcmp(szMagic, "NPRB", 4))) + { + Trace("NnueProbeLoadWeights: %s is not a valid probe weights " + "file (bad magic)\n", szFilename); + fclose(fp); + return FALSE; + } + if ((1 != fread(&uDim, sizeof(uDim), 1, fp)) || + (uDim != 2 * NNUE_ACCUMULATOR_SIZE)) + { + Trace("NnueProbeLoadWeights: %s dimension %lu doesn't match " + "this build's %lu\n", szFilename, uDim, + (ULONG)(2 * NNUE_ACCUMULATOR_SIZE)); + fclose(fp); + return FALSE; + } + +#define READ_OR_FAIL(ptr, count) \ + do { \ + if ((size_t)(count) != fread((ptr), sizeof(float), (count), fp)) \ + { \ + Trace("NnueProbeLoadWeights: %s truncated\n", szFilename); \ + fclose(fp); \ + return FALSE; \ + } \ + } while (0) + + READ_OR_FAIL(g_rgfNnueProbeMu, 2 * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueProbeSigma, 2 * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(g_rgfNnueProbeWeight, 2 * NNUE_ACCUMULATOR_SIZE); + READ_OR_FAIL(&g_fNnueProbeBias, 1); + +#undef READ_OR_FAIL + + fclose(fp); + g_fNnueProbeLoaded = TRUE; + Trace("NnueProbeLoadWeights: loaded %s\n", szFilename); + return TRUE; +} + + +SCORE +NnueProbeScore(SEARCHER_THREAD_CONTEXT *ctx) +/** + +Routine description: + + Cheap lazy-eval proxy: dequantizes+ReLUs the accumulator exactly + like NnueForward's stage 0 (same own-perspective-first + concatenation), standardizes by the probe's fitted mu/sigma, then + a single 512-wide dot product -- no fc1/fc2/fc3 at all. One + multiply-add per dimension plus the dequant/ReLU/standardize work, + versus NnueForward's full ~17,440 MACs across three layers -- + see docs/NNUE.md's latency estimate. + +Parameters: + + SEARCHER_THREAD_CONTEXT *ctx + +Return value: + + SCORE : side-to-move-relative, in centipawns, same convention as + NnueForward (so the two are directly comparable) + +**/ +{ + POSITION *pos = &ctx->sPosition; + ULONG uStm = pos->uToMove; + ULONG uOther = FLIP(uStm); + float fSum; + ULONG i; + + fSum = g_fNnueProbeBias; + for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) + { + float v = (float)ctx->sNnueAccum[uStm].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + v = (v - g_rgfNnueProbeMu[i]) / g_rgfNnueProbeSigma[i]; + fSum += v * g_rgfNnueProbeWeight[i]; + } + for (i = 0; i < NNUE_ACCUMULATOR_SIZE; i++) + { + float v = (float)ctx->sNnueAccum[uOther].rgiValue[i] / g_fNnueFtScale; + v = (v > 0.0f) ? v : 0.0f; + v = (v - g_rgfNnueProbeMu[NNUE_ACCUMULATOR_SIZE + i]) / + g_rgfNnueProbeSigma[NNUE_ACCUMULATOR_SIZE + i]; + fSum += v * g_rgfNnueProbeWeight[NNUE_ACCUMULATOR_SIZE + i]; + } + + return (SCORE)(fSum * 100.0f); +} |
