mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-10-31 07:33:32 +08:00
Merge pull request #1920 from OhOkThisIsFine/claude/focused-herschel-ee8e1c
fix(daemon): contain zombie generations from abandoned requests, name mute endpoint holders
This commit is contained in:
+38
-9
@@ -359,11 +359,20 @@ static cbm_daemon_bootstrap_status_t bootstrap_finish_probe(
|
||||
|
||||
static cbm_daemon_bootstrap_probe_status_t bootstrap_probe(
|
||||
const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops,
|
||||
cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result) {
|
||||
cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result,
|
||||
uint64_t *muted_holder_pid_io) {
|
||||
memset(connect_result, 0, sizeof(*connect_result));
|
||||
*client_out = NULL;
|
||||
return ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms,
|
||||
client_out, connect_result);
|
||||
cbm_daemon_bootstrap_probe_status_t status =
|
||||
ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms,
|
||||
client_out, connect_result);
|
||||
/* Sticky across probes: a later fast-path probe never attempts a connect
|
||||
* and reports no holder, but a mute holder seen once must survive into
|
||||
* the final timeout diagnostic. */
|
||||
if (muted_holder_pid_io && connect_result->muted_endpoint_holder_pid != 0) {
|
||||
*muted_holder_pid_io = connect_result->muted_endpoint_holder_pid;
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
static bool bootstrap_probe_is_finishable(cbm_daemon_bootstrap_probe_status_t probe) {
|
||||
@@ -432,8 +441,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
|
||||
cbm_daemon_runtime_client_t *client = NULL;
|
||||
cbm_daemon_runtime_connect_result_t connect_result;
|
||||
uint64_t muted_holder_pid = 0;
|
||||
cbm_daemon_bootstrap_probe_status_t probe =
|
||||
bootstrap_probe(config, ops, &client, &connect_result);
|
||||
bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
|
||||
if (bootstrap_probe_is_finishable(probe)) {
|
||||
cbm_daemon_bootstrap_status_t status =
|
||||
bootstrap_finish_probe(probe, client, &connect_result, ops, result_out);
|
||||
@@ -461,7 +471,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
* replacements from being launched. */
|
||||
generation_observed = true;
|
||||
bootstrap_pause(deadline);
|
||||
probe = bootstrap_probe(config, ops, &client, &connect_result);
|
||||
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -473,7 +483,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
}
|
||||
if (lock_status == 0) {
|
||||
bootstrap_pause(deadline);
|
||||
probe = bootstrap_probe(config, ops, &client, &connect_result);
|
||||
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
|
||||
continue;
|
||||
}
|
||||
if (lock_status != 1 || !startup_lock) {
|
||||
@@ -482,7 +492,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
}
|
||||
|
||||
lock_acquired = true;
|
||||
probe = bootstrap_probe(config, ops, &client, &connect_result);
|
||||
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
|
||||
if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
|
||||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) {
|
||||
generation_observed = true;
|
||||
@@ -516,7 +526,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
* bootstrap against a daemon that is trying to cleanly stand down. */
|
||||
do {
|
||||
bootstrap_pause(deadline);
|
||||
probe = bootstrap_probe(config, ops, &client, &connect_result);
|
||||
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
|
||||
if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
|
||||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) {
|
||||
generation_observed = true;
|
||||
@@ -545,7 +555,16 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
}
|
||||
|
||||
result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED;
|
||||
if (generation_observed) {
|
||||
if (muted_holder_pid != 0) {
|
||||
/* The one diagnostic the 2026-08-29 zombie recovery had to assemble by
|
||||
* hand from process, pipe, and log correlation: name the pid that
|
||||
* holds the endpoint without answering, and say what to do with it. */
|
||||
(void)snprintf(result_out->message, sizeof(result_out->message),
|
||||
"CBM daemon endpoint is held by pid %llu but that process answered no "
|
||||
"rendezvous within %u ms; the daemon runtime is likely dead — stop that "
|
||||
"process, then retry",
|
||||
(unsigned long long)muted_holder_pid, config->startup_timeout_ms);
|
||||
} else if (generation_observed) {
|
||||
(void)snprintf(result_out->message, sizeof(result_out->message),
|
||||
"CBM daemon is active or starting but could not accept this client "
|
||||
"within %u ms",
|
||||
@@ -578,6 +597,16 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect
|
||||
* existing generation answered. Never reinterpret them as absence. */
|
||||
return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
|
||||
}
|
||||
if (connect_result->status == CBM_DAEMON_RUNTIME_CONNECT_ERROR &&
|
||||
connect_result->muted_endpoint_holder_pid != 0) {
|
||||
/* The transport connected to a live process that answered nothing.
|
||||
* That endpoint is OWNED, whatever the advisory locks read right now
|
||||
* (a wedged generation can hold pipes while its lock files churn).
|
||||
* Classifying this as absence made the 2026-08-29 zombie invisible:
|
||||
* the starter spawned doomed competitors for 30 s and then reported a
|
||||
* bare timeout with no holder named. */
|
||||
return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
|
||||
}
|
||||
if (lifetime_status == 1) {
|
||||
return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
|
||||
}
|
||||
|
||||
+14
-1
@@ -40,6 +40,19 @@ enum {
|
||||
* cancelled and the frontend exits cleanly. */
|
||||
FRONTEND_EOF_DRAIN_MS = 15000,
|
||||
FRONTEND_MAINTENANCE_POLL_MS = 10,
|
||||
/* Cadence for the quiet no-maintenance steady state. Every presence probe
|
||||
* try-acquires the maintenance marker lock, and on Windows that
|
||||
* revalidates the whole ancestor directory chain (a CreateFileW +
|
||||
* identity check per path component). At the old 10 ms cadence this
|
||||
* burned ~20% of a core for the life of every MCP session — 7762
|
||||
* CPU-seconds on one 9.5 h session in the 2026-08-29 incident. The
|
||||
* budget: an activation drains participants within
|
||||
* CLI_ACTIVATION_DRAIN_TIMEOUT_MS (15 s) and the monitor's own exit grace
|
||||
* is FRONTEND_MAINTENANCE_GRACE_MS (3 s), so a 250 ms detection latency
|
||||
* spends under 2% of that window while cutting the probe cost 25-fold.
|
||||
* The 10 ms cadence still paces the monitor's post-detection grace loop,
|
||||
* which does no lock probing. */
|
||||
FRONTEND_MAINTENANCE_IDLE_POLL_MS = 250,
|
||||
/* The owner thread may be draining a supervised process tree. Preserve the
|
||||
* supervisor's complete graceful + forced-settle window before the monitor
|
||||
* fail-stops the process, plus scheduling/teardown margin. */
|
||||
@@ -162,7 +175,7 @@ static void *frontend_maintenance_monitor_worker(void *opaque) {
|
||||
cbm_version_cohort_maintenance_presence_t presence =
|
||||
frontend_observe_maintenance(monitor->manager, true);
|
||||
if (presence == CBM_VERSION_COHORT_MAINTENANCE_ABSENT) {
|
||||
cbm_usleep(FRONTEND_MAINTENANCE_POLL_MS * 1000U);
|
||||
cbm_usleep(FRONTEND_MAINTENANCE_IDLE_POLL_MS * 1000U);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
+95
-3
@@ -36,6 +36,19 @@ static atomic_bool runtime_force_peer_image_mismatch_seam;
|
||||
void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force) {
|
||||
atomic_store(&runtime_force_peer_image_mismatch_seam, force);
|
||||
}
|
||||
/* The abandoned-request containment path ends in process termination, which an
|
||||
* in-process harness cannot observe. The timeout override makes the ceiling
|
||||
* reachable in test time; the hook replaces termination with a recordable
|
||||
* callback. Both stay inert (zero/NULL) outside tests. */
|
||||
static _Atomic uint32_t runtime_abandoned_request_join_timeout_seam;
|
||||
void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms) {
|
||||
atomic_store(&runtime_abandoned_request_join_timeout_seam, timeout_ms);
|
||||
}
|
||||
static _Atomic(cbm_daemon_runtime_containment_hook_t) runtime_containment_hook_seam;
|
||||
void cbm_daemon_runtime_set_containment_hook_for_testing(
|
||||
cbm_daemon_runtime_containment_hook_t hook) {
|
||||
atomic_store(&runtime_containment_hook_seam, hook);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef _WIN32
|
||||
@@ -72,6 +85,16 @@ enum {
|
||||
* a local cooperative peer is already blocked in read and consumes the
|
||||
* rejection within milliseconds. */
|
||||
RUNTIME_REJECT_DRAIN_TIMEOUT_MS = 250,
|
||||
/* Ceiling on how long a disconnecting worker waits for its in-flight
|
||||
* application request to observe cancellation. session_cancel has already
|
||||
* run by the time this wait starts, so a compliant handler returns within
|
||||
* milliseconds; the wait is sized for a handler that only polls its cancel
|
||||
* flag between long pipeline stages. A handler that ignores cancellation
|
||||
* past this ceiling turned the daemon into a permanent zombie once
|
||||
* (2026-08-29): the unbounded join blocked the disconnect path, admission
|
||||
* wedged behind it, and the dead generation held the endpoint pipes and
|
||||
* the UI port for nine hours with nothing logged. */
|
||||
RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS = 30000,
|
||||
RUNTIME_PATH_CAP = 4096,
|
||||
|
||||
RENDEZVOUS_REQUEST_ABI_OFFSET = 0,
|
||||
@@ -1313,6 +1336,26 @@ static void *runtime_application_worker(void *opaque) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static _Noreturn void runtime_cleanup_fail_stop(const char *component);
|
||||
|
||||
static void runtime_contain_unresponsive_application(cbm_daemon_runtime_worker_t *worker) {
|
||||
char peer_pid[32];
|
||||
char token[32];
|
||||
(void)snprintf(peer_pid, sizeof(peer_pid), "%llu", (unsigned long long)worker->peer_process_id);
|
||||
(void)snprintf(token, sizeof(token), "%llu",
|
||||
(unsigned long long)worker->application_request_token);
|
||||
cbm_log_error("daemon.application_request_unresponsive", "peer_pid", peer_pid, "request_token",
|
||||
token);
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
cbm_daemon_runtime_containment_hook_t hook = atomic_load(&runtime_containment_hook_seam);
|
||||
if (hook) {
|
||||
hook("application_request_join");
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
runtime_cleanup_fail_stop("application_request_join");
|
||||
}
|
||||
|
||||
static bool runtime_worker_reap_application(cbm_daemon_runtime_worker_t *worker, bool wait) {
|
||||
if (!worker->application_thread_started) {
|
||||
return true;
|
||||
@@ -1320,6 +1363,35 @@ static bool runtime_worker_reap_application(cbm_daemon_runtime_worker_t *worker,
|
||||
if (!wait && !atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
|
||||
return false;
|
||||
}
|
||||
if (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
|
||||
/* The peer is gone and session_cancel already ran, so the handler owes
|
||||
* a prompt return; only a handler that ignores cancellation reaches
|
||||
* the ceiling. An unbounded join here once turned one wedged request
|
||||
* into a whole-daemon zombie (2026-08-29): the disconnect path
|
||||
* blocked, the worker slot never released, and the dead generation
|
||||
* held the endpoint pipes and UI port for nine hours with nothing
|
||||
* logged while every new client timed out bare. Containment
|
||||
* terminates the process instead — the kernel releases every native
|
||||
* claim and the next client starts a fresh generation. */
|
||||
uint32_t timeout_ms = RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS;
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
uint32_t seam_timeout = atomic_load(&runtime_abandoned_request_join_timeout_seam);
|
||||
if (seam_timeout != 0) {
|
||||
timeout_ms = seam_timeout;
|
||||
}
|
||||
#endif
|
||||
uint64_t deadline = runtime_deadline_after(timeout_ms);
|
||||
while (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
|
||||
if (cbm_now_ms() >= deadline) {
|
||||
/* Terminal in production. A test containment hook returns, and
|
||||
* the join then waits for the harness to release the handler
|
||||
* so teardown stays leak-free under the sanitizers. */
|
||||
runtime_contain_unresponsive_application(worker);
|
||||
deadline = UINT64_MAX;
|
||||
}
|
||||
runtime_wait_tick(deadline);
|
||||
}
|
||||
}
|
||||
if (cbm_thread_join(&worker->application_thread) != 0) {
|
||||
return false;
|
||||
}
|
||||
@@ -2569,8 +2641,11 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi
|
||||
const cbm_daemon_build_identity_t *identity,
|
||||
cbm_daemon_runtime_operation_t operation,
|
||||
uint32_t timeout_ms, uint32_t response_size,
|
||||
uint8_t **payload_out) {
|
||||
uint8_t **payload_out, uint64_t *muted_holder_pid_out) {
|
||||
*payload_out = NULL;
|
||||
if (muted_holder_pid_out) {
|
||||
*muted_holder_pid_out = 0;
|
||||
}
|
||||
if (!endpoint || !identity || !identity->build_fingerprint ||
|
||||
timeout_ms == CBM_DAEMON_IPC_WAIT_FOREVER) {
|
||||
return false;
|
||||
@@ -2590,6 +2665,15 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi
|
||||
int received = sent ? cbm_daemon_ipc_receive_frame_bounded(connection, timeout_ms,
|
||||
response_size, &frame, &payload)
|
||||
: 0;
|
||||
if (muted_holder_pid_out && sent &&
|
||||
(received != 1 || (frame.type == CBM_DAEMON_FRAME_RESPONSE && frame.flags != operation))) {
|
||||
/* Connected but not served: either total silence (dead runtime), or a
|
||||
* wrong-operation reject frame (a wedged generation whose accept path
|
||||
* still answers inline while every worker slot is stuck). Both are
|
||||
* the zombie class — surface the holder's kernel-reported pid so
|
||||
* `daemon status` can name it instead of reporting "not running". */
|
||||
*muted_holder_pid_out = cbm_daemon_ipc_connection_peer_pid(connection);
|
||||
}
|
||||
bool valid = received == 1 && frame.type == CBM_DAEMON_FRAME_RESPONSE &&
|
||||
frame.flags == operation && frame.length == response_size && payload &&
|
||||
payload[0] == 1U;
|
||||
@@ -2617,7 +2701,8 @@ bool cbm_daemon_runtime_request_status(const cbm_daemon_ipc_endpoint_t *endpoint
|
||||
memset(status_out, 0, sizeof(*status_out));
|
||||
uint8_t *payload = NULL;
|
||||
if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STATUS, timeout_ms,
|
||||
CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload)) {
|
||||
CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload,
|
||||
&status_out->muted_endpoint_holder_pid)) {
|
||||
return false;
|
||||
}
|
||||
status_out->permanent = (payload[1] & 0x01U) != 0U;
|
||||
@@ -2648,7 +2733,7 @@ bool cbm_daemon_runtime_request_stop(const cbm_daemon_ipc_endpoint_t *endpoint,
|
||||
memset(result_out, 0, sizeof(*result_out));
|
||||
uint8_t *payload = NULL;
|
||||
if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STOP, timeout_ms,
|
||||
CBM_DAEMON_STOP_RESPONSE_SIZE, &payload)) {
|
||||
CBM_DAEMON_STOP_RESPONSE_SIZE, &payload, NULL)) {
|
||||
return false;
|
||||
}
|
||||
result_out->accepted = (payload[1] & 0x01U) != 0U;
|
||||
@@ -2691,6 +2776,13 @@ cbm_daemon_runtime_client_t *cbm_daemon_runtime_client_connect(
|
||||
frame.length == CBM_DAEMON_RENDEZVOUS_RESPONSE_SIZE &&
|
||||
runtime_hello_response_decode(payload, result_out);
|
||||
free(payload);
|
||||
if (received != 1) {
|
||||
/* The kernel completed the pipe/socket connect, so a server process
|
||||
* exists, yet the HELLO went unanswered. Name that holder: a dead
|
||||
* runtime behind a live endpoint is otherwise indistinguishable from
|
||||
* absence, and the 2026-08-29 zombie hid behind exactly that gap. */
|
||||
result_out->muted_endpoint_holder_pid = cbm_daemon_ipc_connection_peer_pid(connection);
|
||||
}
|
||||
if (!valid || result_out->status != CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED) {
|
||||
cbm_daemon_ipc_connection_close(connection);
|
||||
return NULL;
|
||||
|
||||
@@ -214,6 +214,12 @@ typedef struct {
|
||||
cbm_daemon_client_id_t client_id;
|
||||
/* Kernel-authenticated PID of this client as observed by the daemon. */
|
||||
uint64_t authenticated_process_id;
|
||||
/* Nonzero only when the transport connect succeeded but no valid response
|
||||
* arrived in time: the kernel-reported PID of the process that holds the
|
||||
* endpoint. A held-but-mute endpoint is a live process whose runtime is
|
||||
* wedged (2026-08-29 zombie incident) — callers must report this PID
|
||||
* rather than fold the failure into plain absence. */
|
||||
uint64_t muted_endpoint_holder_pid;
|
||||
cbm_daemon_conflict_t conflict;
|
||||
char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE];
|
||||
} cbm_daemon_runtime_connect_result_t;
|
||||
@@ -265,6 +271,11 @@ typedef struct {
|
||||
uint32_t client_pids[CBM_DAEMON_CONTROL_CLIENT_CAP];
|
||||
char build_fingerprint[CBM_DAEMON_BUILD_FINGERPRINT_SIZE];
|
||||
char semantic_version[CBM_DAEMON_SEMVER_SIZE];
|
||||
/* Meaningful even when the status request itself FAILED: nonzero when the
|
||||
* endpoint accepted the transport connect but answered nothing — the
|
||||
* kernel-reported PID of the mute holder. `daemon status` uses this to
|
||||
* name a zombie generation instead of reporting bare "not running". */
|
||||
uint64_t muted_endpoint_holder_pid;
|
||||
} cbm_daemon_runtime_status_t;
|
||||
|
||||
typedef struct {
|
||||
@@ -421,6 +432,15 @@ void cbm_daemon_runtime_force_peer_image_unverified_for_testing(bool force);
|
||||
* i.e. the tamper case that must still be rejected after unverifiable images
|
||||
* became admissible. */
|
||||
void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force);
|
||||
/* Abandoned-request containment seams (2026-08-29 zombie incident). The
|
||||
* timeout override shrinks the join ceiling to test scale; zero restores the
|
||||
* production constant. The hook replaces the terminal containment stop with a
|
||||
* recordable callback so an in-process harness survives the trigger; NULL
|
||||
* restores process termination. */
|
||||
typedef void (*cbm_daemon_runtime_containment_hook_t)(const char *component);
|
||||
void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms);
|
||||
void cbm_daemon_runtime_set_containment_hook_for_testing(
|
||||
cbm_daemon_runtime_containment_hook_t hook);
|
||||
#endif
|
||||
|
||||
#endif /* CBM_DAEMON_RUNTIME_H */
|
||||
|
||||
@@ -463,7 +463,11 @@ cbm_version_cohort_status_t cbm_version_cohort_acquire(cbm_version_cohort_manage
|
||||
* which a mismatched client can join. Name it: "was the lifetime lock
|
||||
* held?" is exactly what separates a local run (conflict raised) from
|
||||
* a CI run (client admitted), and it was not observable in any log. */
|
||||
cbm_log_info("version_cohort.claimed_unheld", "build",
|
||||
/* Key wording matters here: the old name "claimed_unheld" read as a
|
||||
* stale-claim anomaly and derailed the 2026-08-29 zombie diagnosis,
|
||||
* when the record actually marks the HEALTHY fresh-claim path that
|
||||
* every normal start logs. */
|
||||
cbm_log_info("version_cohort.claimed_fresh", "prior_holder", "none", "build",
|
||||
identity->build_fingerprint ? identity->build_fingerprint : "<null>");
|
||||
status = version_cohort_claim_new(lease, identity, deadline_ms);
|
||||
} else if (status == CBM_VERSION_COHORT_BUSY) {
|
||||
|
||||
+13
@@ -2258,6 +2258,19 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo
|
||||
|
||||
if (strcmp(subcommand, "status") == 0) {
|
||||
if (!active) {
|
||||
if (status.muted_endpoint_holder_pid != 0) {
|
||||
/* Alive process, dead runtime (2026-08-29 zombie class).
|
||||
* "not running" here sent the operator hunting through
|
||||
* processes, pipes, and logs by hand; name the holder and the
|
||||
* recovery instead. */
|
||||
printf("daemon: not responding (endpoint held by pid %llu)\n",
|
||||
(unsigned long long)status.muted_endpoint_holder_pid);
|
||||
printf("hint: that process holds the daemon endpoint but answered nothing; "
|
||||
"its runtime is likely dead. Terminate pid %llu, then run "
|
||||
"`codebase-memory-mcp daemon start`.\n",
|
||||
(unsigned long long)status.muted_endpoint_holder_pid);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
printf("daemon: not running\n");
|
||||
printf("hint: `codebase-memory-mcp daemon start` keeps a daemon warm so CLI "
|
||||
"commands and hooks skip the per-command startup cost.\n");
|
||||
|
||||
@@ -780,6 +780,32 @@ TEST(daemon_bootstrap_rejected_connect_is_reserved_and_never_unavailable) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* 2026-08-29 zombie regression: a connect that reached a live process but got
|
||||
* no answer names that holder in muted_endpoint_holder_pid. That endpoint is
|
||||
* owned regardless of what the advisory locks read, so classification must be
|
||||
* RESERVED on every lifetime answer — never absence, never a spawn license. */
|
||||
TEST(daemon_bootstrap_mute_endpoint_holder_is_reserved_and_never_unavailable) {
|
||||
cbm_daemon_runtime_connect_result_t mute = {0};
|
||||
mute.status = CBM_DAEMON_RUNTIME_CONNECT_ERROR;
|
||||
mute.muted_endpoint_holder_pid = 4242;
|
||||
ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&mute, 1),
|
||||
CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED);
|
||||
ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&mute, 0),
|
||||
CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED);
|
||||
ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&mute, -1),
|
||||
CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED);
|
||||
|
||||
/* A protocol-level rejection still wins over the holder pid: an answering
|
||||
* generation is more precise evidence than a silent one. */
|
||||
cbm_daemon_runtime_connect_result_t rejected = {0};
|
||||
rejected.status = CBM_DAEMON_RUNTIME_CONNECT_REJECTED;
|
||||
rejected.muted_endpoint_holder_pid = 4242;
|
||||
snprintf(rejected.message, sizeof(rejected.message), "CBM daemon is stopping");
|
||||
ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&rejected, 0),
|
||||
CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL);
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(daemon_bootstrap_concurrent_first_clients_spawn_one_daemon) {
|
||||
bootstrap_endpoint_fixture_t fixture;
|
||||
ASSERT_TRUE(bootstrap_endpoint_fixture_start(&fixture, "startup-race"));
|
||||
@@ -872,6 +898,7 @@ SUITE(daemon_bootstrap) {
|
||||
RUN_TEST(daemon_bootstrap_reserved_then_absent_spawns_replacement);
|
||||
RUN_TEST(daemon_bootstrap_releases_handoff_when_spawned_generation_is_reserved);
|
||||
RUN_TEST(daemon_bootstrap_rejected_connect_is_reserved_and_never_unavailable);
|
||||
RUN_TEST(daemon_bootstrap_mute_endpoint_holder_is_reserved_and_never_unavailable);
|
||||
RUN_TEST(daemon_bootstrap_concurrent_first_clients_spawn_one_daemon);
|
||||
#ifdef __APPLE__
|
||||
RUN_TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous);
|
||||
|
||||
@@ -3946,6 +3946,224 @@ TEST(daemon_runtime_noncooperative_callback_does_not_detach_or_unbound_stop) {
|
||||
PASS();
|
||||
}
|
||||
|
||||
#if defined(CBM_ENABLE_TEST_SEAMS)
|
||||
static atomic_int runtime_test_containment_fired;
|
||||
static atomic_bool runtime_test_containment_component_exact;
|
||||
|
||||
static void runtime_test_containment_hook(const char *component) {
|
||||
if (component && strcmp(component, "application_request_join") == 0) {
|
||||
atomic_store_explicit(&runtime_test_containment_component_exact, true,
|
||||
memory_order_release);
|
||||
}
|
||||
atomic_fetch_add_explicit(&runtime_test_containment_fired, 1, memory_order_release);
|
||||
}
|
||||
|
||||
/* 2026-08-29 zombie regression: a disconnecting worker joined its in-flight
|
||||
* application request with no deadline. A handler that ignored cancellation
|
||||
* therefore wedged the disconnect path forever and the dead generation kept
|
||||
* the endpoint. The join must now reach the containment boundary in bounded
|
||||
* time; the seam hook stands in for the terminal process stop so the harness
|
||||
* can then supply cooperation and prove a clean join and teardown. */
|
||||
TEST(daemon_runtime_abandoned_request_join_reaches_containment_in_bounded_time) {
|
||||
static const uint8_t request[] = {'w', 'e', 'd', 'g', 'e'};
|
||||
enum { JOIN_BOUND_MS = 150, CONTAINMENT_OBSERVED_MAX_MS = 5000 };
|
||||
cbm_daemon_build_identity_t identity =
|
||||
runtime_test_identity("2.4.0", runtime_test_self_build());
|
||||
runtime_application_context_t context;
|
||||
runtime_application_context_init(&context, true);
|
||||
atomic_store_explicit(&context.ignore_first_request_cancel, true, memory_order_release);
|
||||
atomic_store(&runtime_test_containment_fired, 0);
|
||||
atomic_store(&runtime_test_containment_component_exact, false);
|
||||
atomic_bool request_thread_completed;
|
||||
atomic_init(&request_thread_completed, false);
|
||||
runtime_test_fixture_t fixture;
|
||||
bool started = runtime_test_fixture_start_application(&fixture, "application-abandoned-join",
|
||||
&identity, &context);
|
||||
cbm_daemon_runtime_connect_result_t result = {0};
|
||||
cbm_daemon_runtime_client_t *client = NULL;
|
||||
runtime_application_client_call_t call = {
|
||||
.request = request,
|
||||
.request_length = (uint32_t)sizeof(request),
|
||||
.completed = &request_thread_completed,
|
||||
.status = CBM_DAEMON_RUNTIME_APPLICATION_OK,
|
||||
};
|
||||
cbm_thread_t request_thread;
|
||||
int request_thread_create_rc = -1;
|
||||
int request_thread_join_rc = -1;
|
||||
bool request_thread_started = false;
|
||||
bool callback_started = false;
|
||||
bool close_begun = false;
|
||||
bool containment_observed = false;
|
||||
uint64_t containment_elapsed_ms = UINT64_MAX;
|
||||
bool exited_after_release = false;
|
||||
|
||||
cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(JOIN_BOUND_MS);
|
||||
cbm_daemon_runtime_set_containment_hook_for_testing(runtime_test_containment_hook);
|
||||
|
||||
if (started) {
|
||||
client = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity,
|
||||
RUNTIME_TEST_TIMEOUT_MS, &result);
|
||||
}
|
||||
if (client) {
|
||||
call.client = client;
|
||||
request_thread_create_rc = cbm_thread_create(
|
||||
&request_thread, 128U * 1024U, runtime_application_client_request_thread, &call);
|
||||
request_thread_started = request_thread_create_rc == 0;
|
||||
callback_started =
|
||||
request_thread_started &&
|
||||
runtime_test_wait_atomic_bool(&context.first_request_started, RUNTIME_TEST_TIMEOUT_MS);
|
||||
}
|
||||
if (callback_started) {
|
||||
uint64_t containment_started_ms = cbm_now_ms();
|
||||
close_begun = cbm_daemon_runtime_client_close_begin(client);
|
||||
uint64_t deadline = cbm_now_ms() + CONTAINMENT_OBSERVED_MAX_MS;
|
||||
while (atomic_load_explicit(&runtime_test_containment_fired, memory_order_acquire) == 0 &&
|
||||
cbm_now_ms() < deadline) {
|
||||
struct timespec pause = {.tv_sec = 0, .tv_nsec = 1000000};
|
||||
(void)cbm_nanosleep(&pause, NULL);
|
||||
}
|
||||
containment_observed =
|
||||
atomic_load_explicit(&runtime_test_containment_fired, memory_order_acquire) > 0;
|
||||
containment_elapsed_ms = cbm_now_ms() - containment_started_ms;
|
||||
}
|
||||
|
||||
/* Cooperation after the boundary: the reap waits for the released handler
|
||||
* so join and teardown stay provably leak-free under the sanitizers. */
|
||||
atomic_store_explicit(&context.release_first_request, true, memory_order_release);
|
||||
if (request_thread_started) {
|
||||
request_thread_join_rc = cbm_thread_join(&request_thread);
|
||||
}
|
||||
if (client) {
|
||||
(void)cbm_daemon_runtime_client_close_finish(client, RUNTIME_TEST_TIMEOUT_MS);
|
||||
client = NULL;
|
||||
}
|
||||
if (started) {
|
||||
exited_after_release =
|
||||
cbm_daemon_runtime_service_wait_exited(fixture.service, RUNTIME_TEST_TIMEOUT_MS);
|
||||
}
|
||||
free(call.response);
|
||||
runtime_test_fixture_finish(&fixture);
|
||||
cbm_daemon_runtime_set_containment_hook_for_testing(NULL);
|
||||
cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(0);
|
||||
|
||||
ASSERT_TRUE(started);
|
||||
ASSERT_EQ(result.status, CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED);
|
||||
ASSERT_EQ(request_thread_create_rc, 0);
|
||||
ASSERT_TRUE(callback_started);
|
||||
ASSERT_TRUE(close_begun);
|
||||
ASSERT_TRUE(containment_observed);
|
||||
ASSERT_TRUE(containment_elapsed_ms <= CONTAINMENT_OBSERVED_MAX_MS);
|
||||
ASSERT_EQ(atomic_load(&runtime_test_containment_fired), 1);
|
||||
ASSERT_TRUE(atomic_load(&runtime_test_containment_component_exact));
|
||||
ASSERT_EQ(request_thread_join_rc, 0);
|
||||
ASSERT_TRUE(exited_after_release);
|
||||
ASSERT_EQ(atomic_load(&context.opened), 1);
|
||||
ASSERT_EQ(atomic_load(&context.requests), 1);
|
||||
ASSERT_EQ(atomic_load(&context.cancelled), 1);
|
||||
ASSERT_EQ(atomic_load(&context.closed), 1);
|
||||
PASS();
|
||||
}
|
||||
#endif
|
||||
|
||||
typedef struct {
|
||||
cbm_daemon_ipc_listener_t *listener;
|
||||
cbm_daemon_ipc_connection_t *held[2];
|
||||
atomic_int held_count;
|
||||
atomic_bool stop;
|
||||
} runtime_mute_holder_t;
|
||||
|
||||
/* Accepts up to two transport connections and never answers a frame: the
|
||||
* kernel-level shape of the 2026-08-29 zombie (endpoint owned, runtime dead). */
|
||||
static void *runtime_test_mute_holder_thread(void *opaque) {
|
||||
runtime_mute_holder_t *holder = opaque;
|
||||
while (!atomic_load_explicit(&holder->stop, memory_order_acquire) &&
|
||||
atomic_load_explicit(&holder->held_count, memory_order_acquire) < 2) {
|
||||
cbm_daemon_ipc_connection_t *connection = NULL;
|
||||
int accepted = cbm_daemon_ipc_accept(holder->listener, 50, &connection);
|
||||
if (accepted == 1 && connection) {
|
||||
int index = atomic_load_explicit(&holder->held_count, memory_order_acquire);
|
||||
holder->held[index] = connection;
|
||||
atomic_store_explicit(&holder->held_count, index + 1, memory_order_release);
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static uint64_t runtime_test_self_process_id(void) {
|
||||
#ifdef _WIN32
|
||||
return (uint64_t)GetCurrentProcessId();
|
||||
#else
|
||||
return (uint64_t)getpid();
|
||||
#endif
|
||||
}
|
||||
|
||||
/* 2026-08-29 zombie regression: a held endpoint that answers nothing must
|
||||
* name its holder. Both the HELLO connect and the one-shot status probe
|
||||
* fail against a mute holder, and both must report the holder's
|
||||
* kernel-authenticated pid instead of plain absence. */
|
||||
TEST(daemon_runtime_mute_endpoint_holder_pid_is_reported) {
|
||||
enum { MUTE_CONNECT_TIMEOUT_MS = 500 };
|
||||
cbm_daemon_build_identity_t identity =
|
||||
runtime_test_identity("2.4.0", runtime_test_self_build());
|
||||
char parent[RUNTIME_TEST_PATH_CAP] = {0};
|
||||
char key[CBM_DAEMON_KEY_SIZE] = {0};
|
||||
char runtime_dir[RUNTIME_TEST_PATH_CAP] = {0};
|
||||
cbm_daemon_ipc_endpoint_t *endpoint = NULL;
|
||||
runtime_mute_holder_t holder = {0};
|
||||
atomic_init(&holder.held_count, 0);
|
||||
atomic_init(&holder.stop, false);
|
||||
cbm_thread_t holder_thread;
|
||||
bool holder_started = false;
|
||||
|
||||
bool prepared = th_secure_runtime_parent_new(parent, sizeof(parent), "mute-holder") &&
|
||||
cbm_daemon_rendezvous_key(key);
|
||||
endpoint = prepared ? cbm_daemon_ipc_endpoint_new(key, parent) : NULL;
|
||||
bool runtime_dir_copied =
|
||||
endpoint &&
|
||||
runtime_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint));
|
||||
holder.listener = endpoint ? cbm_daemon_ipc_listen(endpoint) : NULL;
|
||||
holder_started =
|
||||
holder.listener &&
|
||||
cbm_thread_create(&holder_thread, 0, runtime_test_mute_holder_thread, &holder) == 0;
|
||||
|
||||
cbm_daemon_runtime_connect_result_t connect_result = {0};
|
||||
cbm_daemon_runtime_client_t *client = NULL;
|
||||
cbm_daemon_runtime_status_t status = {0};
|
||||
bool status_active = true;
|
||||
if (holder_started) {
|
||||
client = cbm_daemon_runtime_client_connect(endpoint, &identity, MUTE_CONNECT_TIMEOUT_MS,
|
||||
&connect_result);
|
||||
status_active = cbm_daemon_runtime_request_status(endpoint, &identity,
|
||||
MUTE_CONNECT_TIMEOUT_MS, &status);
|
||||
}
|
||||
|
||||
atomic_store_explicit(&holder.stop, true, memory_order_release);
|
||||
if (holder_started) {
|
||||
(void)cbm_thread_join(&holder_thread);
|
||||
}
|
||||
for (size_t index = 0; index < 2; index++) {
|
||||
if (holder.held[index]) {
|
||||
cbm_daemon_ipc_connection_close(holder.held[index]);
|
||||
}
|
||||
}
|
||||
if (holder.listener) {
|
||||
cbm_daemon_ipc_listener_close(holder.listener);
|
||||
}
|
||||
cbm_daemon_ipc_endpoint_free(endpoint);
|
||||
(void)cbm_rmdir(runtime_dir);
|
||||
(void)cbm_rmdir(parent);
|
||||
|
||||
ASSERT_TRUE(prepared);
|
||||
ASSERT_TRUE(runtime_dir_copied);
|
||||
ASSERT_TRUE(holder_started);
|
||||
ASSERT_NULL(client);
|
||||
ASSERT_EQ(connect_result.status, CBM_DAEMON_RUNTIME_CONNECT_ERROR);
|
||||
ASSERT_EQ(connect_result.muted_endpoint_holder_pid, runtime_test_self_process_id());
|
||||
ASSERT_FALSE(status_active);
|
||||
ASSERT_EQ(status.muted_endpoint_holder_pid, runtime_test_self_process_id());
|
||||
PASS();
|
||||
}
|
||||
|
||||
TEST(daemon_runtime_application_busy_cap_and_malformed_are_isolated) {
|
||||
static const uint8_t blocking_request[] = {'f', 'i', 'r', 's', 't'};
|
||||
static const uint8_t busy_request[] = {'b', 'u', 's', 'y'};
|
||||
@@ -4795,6 +5013,10 @@ SUITE(daemon_runtime) {
|
||||
RUN_TEST(daemon_runtime_close_begin_releases_admission_with_inflight_request);
|
||||
RUN_TEST(daemon_runtime_disconnect_cancels_blocked_non_index_child_and_preserves_other_session);
|
||||
RUN_TEST(daemon_runtime_noncooperative_callback_does_not_detach_or_unbound_stop);
|
||||
#if defined(CBM_ENABLE_TEST_SEAMS)
|
||||
RUN_TEST(daemon_runtime_abandoned_request_join_reaches_containment_in_bounded_time);
|
||||
#endif
|
||||
RUN_TEST(daemon_runtime_mute_endpoint_holder_pid_is_reported);
|
||||
RUN_TEST(daemon_runtime_application_busy_cap_and_malformed_are_isolated);
|
||||
RUN_TEST(daemon_runtime_malformed_and_zero_cancel_close_only_offending_connections);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user