wip: windows region census to attribute committed memory by pool

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

src/foundation/mem.c

@@ -21,6 +21,7 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 #ifdef _WIN32
 #ifndef WIN32_LEAN_AND_MEAN
@@ -707,3 +708,86 @@ int cbm_mem_phase_report_json(char *out, size_t size) {
     }
     return length > 0 && (size_t)length < size ? length : 0;
 }
+
+/* ── Windows region census (see mem.h) ─────────────────────────────── */
+
+static atomic_int g_mem_census_enabled = -1;
+
+bool cbm_mem_census_enabled(void) {
+    int state = atomic_load_explicit(&g_mem_census_enabled, memory_order_acquire);
+    if (state >= 0) {
+        return state == 1;
+    }
+    char buf[CBM_SZ_16];
+    bool on = cbm_safe_getenv("CBM_MEM_CENSUS", buf, sizeof(buf), NULL) != NULL && buf[0] == '1';
+    atomic_store_explicit(&g_mem_census_enabled, on ? 1 : 0, memory_order_release);
+    return on;
+}
+
+bool cbm_mem_win_census(cbm_mem_win_census_t *out) {
+    if (!out) {
+        return false;
+    }
+    memset(out, 0, sizeof(*out));
+#ifndef _WIN32
+    return false;
+#else
+    /* Walk the whole user address space once. VirtualQuery reports the largest
+     * run of pages sharing a state/type, so the region count also tells us how
+     * fragmented the space has become. */
+    MEMORY_BASIC_INFORMATION mbi;
+    unsigned char *address = NULL;
+    while (VirtualQuery(address, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+        out->regions++;
+        if (mbi.State == MEM_COMMIT) {
+            out->committed_total += mbi.RegionSize;
+            switch (mbi.Type) {
+            case MEM_PRIVATE:
+                out->committed_private += mbi.RegionSize;
+                break;
+            case MEM_MAPPED:
+                out->committed_mapped += mbi.RegionSize;
+                break;
+            case MEM_IMAGE:
+                out->committed_image += mbi.RegionSize;
+                break;
+            default:
+                break;
+            }
+        } else if (mbi.State == MEM_RESERVE) {
+            out->reserved_total += mbi.RegionSize;
+        }
+        unsigned char *next = (unsigned char *)mbi.BaseAddress + mbi.RegionSize;
+        if (next <= address) {
+            break; /* no forward progress: stop rather than spin */
+        }
+        address = next;
+    }
+
+    /* The CRT heaps are the pool --wrap cannot reach: anything ucrtbase or a
+     * system DLL allocates internally lands here, and Windows heaps do not
+     * decommit on free. HeapWalk needs the heap locked to be coherent. */
+    HANDLE heaps[64];
+    DWORD count = GetProcessHeaps((DWORD)(sizeof(heaps) / sizeof(heaps[0])), heaps);
+    if (count > 0) {
+        out->heap_walk_ok = true;
+        out->crt_heap_count = (unsigned)(count > 64 ? 64 : count);
+        for (DWORD i = 0; i < out->crt_heap_count; i++) {
+            if (!HeapLock(heaps[i])) {
+                continue;
+            }
+            PROCESS_HEAP_ENTRY entry;
+            memset(&entry, 0, sizeof(entry));
+            while (HeapWalk(heaps[i], &entry)) {
+                if ((entry.wFlags & PROCESS_HEAP_REGION) != 0) {
+                    out->crt_heap_committed += entry.Region.dwCommittedSize;
+                } else if ((entry.wFlags & PROCESS_HEAP_ENTRY_BUSY) != 0) {
+                    out->crt_heap_busy += entry.cbData + entry.cbOverhead;
+                }
+            }
+            HeapUnlock(heaps[i]);
+        }
+    }
+    return true;
+#endif
+}

src/foundation/mem.h

@@ -200,4 +200,48 @@ void cbm_mem_phase_reset(void);
  * first. Returns bytes written (0 when disabled or empty). */
 int cbm_mem_phase_report_json(char *out, size_t size);
 
+/* ── Windows region census: which POOL holds the memory? ─────────────
+ *
+ * The allocator walk can only describe memory the allocator owns. When the
+ * process commits far more than the allocator holds — 440 MB private against
+ * 34.7 MiB of arenas (#581) — the remainder is by definition somewhere else,
+ * and "somewhere else" has to be named before it can be fixed. Guessing
+ * between the CRT heap, private VirtualAlloc, mapped files and thread stacks
+ * costs a soak run per guess; asking the OS costs one pass.
+ *
+ * Two independent walks, deliberately overlapping so they cross-check:
+ *   VirtualQuery over the whole address space  → committed bytes by region TYPE
+ *   GetProcessHeaps + HeapWalk                 → what the CRT heaps hold
+ *
+ * committed_private is the total the OS charges us for anonymous memory; the
+ * allocator's arenas and the CRT heaps are both drawn from it, so
+ *   committed_private - crt_heap_committed - <arena committed>
+ * is the unattributed remainder. A census where that remainder is the part
+ * that grows says the leak is in neither heap — which is a different bug from
+ * either heap leaking, and points at raw VirtualAlloc or stacks instead.
+ *
+ * Windows-only; returns false everywhere else so callers must handle absence
+ * rather than silently reading zeros as "nothing there".
+ */
+typedef struct {
+    size_t committed_total;   /* all MEM_COMMIT regions */
+    size_t committed_private; /* MEM_PRIVATE: heaps, stacks, raw VirtualAlloc */
+    size_t committed_mapped;  /* MEM_MAPPED: file/section views */
+    size_t committed_image;   /* MEM_IMAGE: loaded modules */
+    size_t reserved_total;    /* MEM_RESERVE, uncommitted */
+    size_t crt_heap_committed; /* committed bytes across all CRT heaps */
+    size_t crt_heap_busy;      /* bytes in live allocations inside them */
+    unsigned crt_heap_count;
+    unsigned regions;
+    bool heap_walk_ok; /* false → crt_* are unknown, not zero */
+} cbm_mem_win_census_t;
+
+/* Snapshot the OS-level region census. Returns false when unavailable
+ * (non-Windows, or out is NULL). */
+bool cbm_mem_win_census(cbm_mem_win_census_t *out);
+
+/* Census logging gate: CBM_MEM_CENSUS=1. Off by default — HeapWalk locks each
+ * CRT heap, so this is a diagnostic, never a hot-path metric. */
+bool cbm_mem_census_enabled(void);
+
 #endif /* CBM_MEM_H */

src/mcp/mcp.c

@@ -10449,6 +10449,34 @@ char *cbm_mcp_handle_tool(cbm_mcp_server_t *srv, const char *tool_name, const ch
     cbm_mem_phase_mark("request.release_store");
     release_request_store(srv);
     cbm_mem_phase_mark("idle");
+    /* One census per completed request, so growth can be attributed to a POOL
+     * rather than inferred from a process total (#581). Emitted after the
+     * request store is released, which is the point where a well-behaved
+     * request has given everything back. */
+    if (cbm_mem_census_enabled()) {
+        cbm_mem_win_census_t census;
+        if (cbm_mem_win_census(&census)) {
+            const size_t mb = (size_t)CBM_SZ_1K * CBM_SZ_1K;
+            char private_mb[CBM_SZ_32];
+            char crt_mb[CBM_SZ_32];
+            char busy_mb[CBM_SZ_32];
+            char mapped_mb[CBM_SZ_32];
+            char rss_mb[CBM_SZ_32];
+            char regions[CBM_SZ_32];
+            (void)snprintf(private_mb, sizeof(private_mb), "%zu",
+                           census.committed_private / mb);
+            (void)snprintf(crt_mb, sizeof(crt_mb), "%zu",
+                           census.crt_heap_committed / mb);
+            (void)snprintf(busy_mb, sizeof(busy_mb), "%zu", census.crt_heap_busy / mb);
+            (void)snprintf(mapped_mb, sizeof(mapped_mb), "%zu",
+                           census.committed_mapped / mb);
+            (void)snprintf(rss_mb, sizeof(rss_mb), "%zu", cbm_mem_rss() / mb);
+            (void)snprintf(regions, sizeof(regions), "%u", census.regions);
+            cbm_log_info("mem.census", "private_mb", private_mb, "crt_heap_mb", crt_mb,
+                         "crt_busy_mb", busy_mb, "mapped_mb", mapped_mb, "rss_mb", rss_mb,
+                         "regions", regions);
+        }
+    }
     return result;
 }