Merge pull request #1920 from OhOkThisIsFine/claude/focused-herschel-ee8e1c

fix(daemon): contain zombie generations from abandoned requests, name mute endpoint holders
This commit is contained in:
Martin Vogel
2026-08-31 14:56:11 +02:00
committed by GitHub
8 changed files with 434 additions and 14 deletions
+38 -9
View File
@@ -359,11 +359,20 @@ static cbm_daemon_bootstrap_status_t bootstrap_finish_probe(
static cbm_daemon_bootstrap_probe_status_t bootstrap_probe(
const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops,
cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result) {
cbm_daemon_runtime_client_t **client_out, cbm_daemon_runtime_connect_result_t *connect_result,
uint64_t *muted_holder_pid_io) {
memset(connect_result, 0, sizeof(*connect_result));
*client_out = NULL;
return ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms,
client_out, connect_result);
cbm_daemon_bootstrap_probe_status_t status =
ops->probe(ops->context, config->endpoint, config->identity, config->connect_timeout_ms,
client_out, connect_result);
/* Sticky across probes: a later fast-path probe never attempts a connect
* and reports no holder, but a mute holder seen once must survive into
* the final timeout diagnostic. */
if (muted_holder_pid_io && connect_result->muted_endpoint_holder_pid != 0) {
*muted_holder_pid_io = connect_result->muted_endpoint_holder_pid;
}
return status;
}
static bool bootstrap_probe_is_finishable(cbm_daemon_bootstrap_probe_status_t probe) {
@@ -432,8 +441,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
cbm_daemon_runtime_client_t *client = NULL;
cbm_daemon_runtime_connect_result_t connect_result;
uint64_t muted_holder_pid = 0;
cbm_daemon_bootstrap_probe_status_t probe =
bootstrap_probe(config, ops, &client, &connect_result);
bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
if (bootstrap_probe_is_finishable(probe)) {
cbm_daemon_bootstrap_status_t status =
bootstrap_finish_probe(probe, client, &connect_result, ops, result_out);
@@ -461,7 +471,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
* replacements from being launched. */
generation_observed = true;
bootstrap_pause(deadline);
probe = bootstrap_probe(config, ops, &client, &connect_result);
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
continue;
}
@@ -473,7 +483,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
}
if (lock_status == 0) {
bootstrap_pause(deadline);
probe = bootstrap_probe(config, ops, &client, &connect_result);
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
continue;
}
if (lock_status != 1 || !startup_lock) {
@@ -482,7 +492,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
}
lock_acquired = true;
probe = bootstrap_probe(config, ops, &client, &connect_result);
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) {
generation_observed = true;
@@ -516,7 +526,7 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
* bootstrap against a daemon that is trying to cleanly stand down. */
do {
bootstrap_pause(deadline);
probe = bootstrap_probe(config, ops, &client, &connect_result);
probe = bootstrap_probe(config, ops, &client, &connect_result, &muted_holder_pid);
if (probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL) {
generation_observed = true;
@@ -545,7 +555,16 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
}
result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED;
if (generation_observed) {
if (muted_holder_pid != 0) {
/* The one diagnostic the 2026-08-29 zombie recovery had to assemble by
* hand from process, pipe, and log correlation: name the pid that
* holds the endpoint without answering, and say what to do with it. */
(void)snprintf(result_out->message, sizeof(result_out->message),
"CBM daemon endpoint is held by pid %llu but that process answered no "
"rendezvous within %u ms; the daemon runtime is likely dead — stop that "
"process, then retry",
(unsigned long long)muted_holder_pid, config->startup_timeout_ms);
} else if (generation_observed) {
(void)snprintf(result_out->message, sizeof(result_out->message),
"CBM daemon is active or starting but could not accept this client "
"within %u ms",
@@ -578,6 +597,16 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect
* existing generation answered. Never reinterpret them as absence. */
return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
}
if (connect_result->status == CBM_DAEMON_RUNTIME_CONNECT_ERROR &&
connect_result->muted_endpoint_holder_pid != 0) {
/* The transport connected to a live process that answered nothing.
* That endpoint is OWNED, whatever the advisory locks read right now
* (a wedged generation can hold pipes while its lock files churn).
* Classifying this as absence made the 2026-08-29 zombie invisible:
* the starter spawned doomed competitors for 30 s and then reported a
* bare timeout with no holder named. */
return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
}
if (lifetime_status == 1) {
return CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED;
}
+14 -1
View File
@@ -40,6 +40,19 @@ enum {
* cancelled and the frontend exits cleanly. */
FRONTEND_EOF_DRAIN_MS = 15000,
FRONTEND_MAINTENANCE_POLL_MS = 10,
/* Cadence for the quiet no-maintenance steady state. Every presence probe
* try-acquires the maintenance marker lock, and on Windows that
* revalidates the whole ancestor directory chain (a CreateFileW +
* identity check per path component). At the old 10 ms cadence this
* burned ~20% of a core for the life of every MCP session — 7762
* CPU-seconds on one 9.5 h session in the 2026-08-29 incident. The
* budget: an activation drains participants within
* CLI_ACTIVATION_DRAIN_TIMEOUT_MS (15 s) and the monitor's own exit grace
* is FRONTEND_MAINTENANCE_GRACE_MS (3 s), so a 250 ms detection latency
* spends under 2% of that window while cutting the probe cost 25-fold.
* The 10 ms cadence still paces the monitor's post-detection grace loop,
* which does no lock probing. */
FRONTEND_MAINTENANCE_IDLE_POLL_MS = 250,
/* The owner thread may be draining a supervised process tree. Preserve the
* supervisor's complete graceful + forced-settle window before the monitor
* fail-stops the process, plus scheduling/teardown margin. */
@@ -162,7 +175,7 @@ static void *frontend_maintenance_monitor_worker(void *opaque) {
cbm_version_cohort_maintenance_presence_t presence =
frontend_observe_maintenance(monitor->manager, true);
if (presence == CBM_VERSION_COHORT_MAINTENANCE_ABSENT) {
cbm_usleep(FRONTEND_MAINTENANCE_POLL_MS * 1000U);
cbm_usleep(FRONTEND_MAINTENANCE_IDLE_POLL_MS * 1000U);
continue;
}
+95 -3
View File
@@ -36,6 +36,19 @@ static atomic_bool runtime_force_peer_image_mismatch_seam;
void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force) {
atomic_store(&runtime_force_peer_image_mismatch_seam, force);
}
/* The abandoned-request containment path ends in process termination, which an
* in-process harness cannot observe. The timeout override makes the ceiling
* reachable in test time; the hook replaces termination with a recordable
* callback. Both stay inert (zero/NULL) outside tests. */
static _Atomic uint32_t runtime_abandoned_request_join_timeout_seam;
void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms) {
atomic_store(&runtime_abandoned_request_join_timeout_seam, timeout_ms);
}
static _Atomic(cbm_daemon_runtime_containment_hook_t) runtime_containment_hook_seam;
void cbm_daemon_runtime_set_containment_hook_for_testing(
cbm_daemon_runtime_containment_hook_t hook) {
atomic_store(&runtime_containment_hook_seam, hook);
}
#endif
#ifdef _WIN32
@@ -72,6 +85,16 @@ enum {
* a local cooperative peer is already blocked in read and consumes the
* rejection within milliseconds. */
RUNTIME_REJECT_DRAIN_TIMEOUT_MS = 250,
/* Ceiling on how long a disconnecting worker waits for its in-flight
* application request to observe cancellation. session_cancel has already
* run by the time this wait starts, so a compliant handler returns within
* milliseconds; the wait is sized for a handler that only polls its cancel
* flag between long pipeline stages. A handler that ignores cancellation
* past this ceiling turned the daemon into a permanent zombie once
* (2026-08-29): the unbounded join blocked the disconnect path, admission
* wedged behind it, and the dead generation held the endpoint pipes and
* the UI port for nine hours with nothing logged. */
RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS = 30000,
RUNTIME_PATH_CAP = 4096,
RENDEZVOUS_REQUEST_ABI_OFFSET = 0,
@@ -1313,6 +1336,26 @@ static void *runtime_application_worker(void *opaque) {
return NULL;
}
static _Noreturn void runtime_cleanup_fail_stop(const char *component);
static void runtime_contain_unresponsive_application(cbm_daemon_runtime_worker_t *worker) {
char peer_pid[32];
char token[32];
(void)snprintf(peer_pid, sizeof(peer_pid), "%llu", (unsigned long long)worker->peer_process_id);
(void)snprintf(token, sizeof(token), "%llu",
(unsigned long long)worker->application_request_token);
cbm_log_error("daemon.application_request_unresponsive", "peer_pid", peer_pid, "request_token",
token);
#ifdef CBM_ENABLE_TEST_SEAMS
cbm_daemon_runtime_containment_hook_t hook = atomic_load(&runtime_containment_hook_seam);
if (hook) {
hook("application_request_join");
return;
}
#endif
runtime_cleanup_fail_stop("application_request_join");
}
static bool runtime_worker_reap_application(cbm_daemon_runtime_worker_t *worker, bool wait) {
if (!worker->application_thread_started) {
return true;
@@ -1320,6 +1363,35 @@ static bool runtime_worker_reap_application(cbm_daemon_runtime_worker_t *worker,
if (!wait && !atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
return false;
}
if (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
/* The peer is gone and session_cancel already ran, so the handler owes
* a prompt return; only a handler that ignores cancellation reaches
* the ceiling. An unbounded join here once turned one wedged request
* into a whole-daemon zombie (2026-08-29): the disconnect path
* blocked, the worker slot never released, and the dead generation
* held the endpoint pipes and UI port for nine hours with nothing
* logged while every new client timed out bare. Containment
* terminates the process instead — the kernel releases every native
* claim and the next client starts a fresh generation. */
uint32_t timeout_ms = RUNTIME_ABANDONED_REQUEST_JOIN_TIMEOUT_MS;
#ifdef CBM_ENABLE_TEST_SEAMS
uint32_t seam_timeout = atomic_load(&runtime_abandoned_request_join_timeout_seam);
if (seam_timeout != 0) {
timeout_ms = seam_timeout;
}
#endif
uint64_t deadline = runtime_deadline_after(timeout_ms);
while (!atomic_load_explicit(&worker->application_thread_done, memory_order_acquire)) {
if (cbm_now_ms() >= deadline) {
/* Terminal in production. A test containment hook returns, and
* the join then waits for the harness to release the handler
* so teardown stays leak-free under the sanitizers. */
runtime_contain_unresponsive_application(worker);
deadline = UINT64_MAX;
}
runtime_wait_tick(deadline);
}
}
if (cbm_thread_join(&worker->application_thread) != 0) {
return false;
}
@@ -2569,8 +2641,11 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi
const cbm_daemon_build_identity_t *identity,
cbm_daemon_runtime_operation_t operation,
uint32_t timeout_ms, uint32_t response_size,
uint8_t **payload_out) {
uint8_t **payload_out, uint64_t *muted_holder_pid_out) {
*payload_out = NULL;
if (muted_holder_pid_out) {
*muted_holder_pid_out = 0;
}
if (!endpoint || !identity || !identity->build_fingerprint ||
timeout_ms == CBM_DAEMON_IPC_WAIT_FOREVER) {
return false;
@@ -2590,6 +2665,15 @@ static bool runtime_control_request_send(const cbm_daemon_ipc_endpoint_t *endpoi
int received = sent ? cbm_daemon_ipc_receive_frame_bounded(connection, timeout_ms,
response_size, &frame, &payload)
: 0;
if (muted_holder_pid_out && sent &&
(received != 1 || (frame.type == CBM_DAEMON_FRAME_RESPONSE && frame.flags != operation))) {
/* Connected but not served: either total silence (dead runtime), or a
* wrong-operation reject frame (a wedged generation whose accept path
* still answers inline while every worker slot is stuck). Both are
* the zombie class — surface the holder's kernel-reported pid so
* `daemon status` can name it instead of reporting "not running". */
*muted_holder_pid_out = cbm_daemon_ipc_connection_peer_pid(connection);
}
bool valid = received == 1 && frame.type == CBM_DAEMON_FRAME_RESPONSE &&
frame.flags == operation && frame.length == response_size && payload &&
payload[0] == 1U;
@@ -2617,7 +2701,8 @@ bool cbm_daemon_runtime_request_status(const cbm_daemon_ipc_endpoint_t *endpoint
memset(status_out, 0, sizeof(*status_out));
uint8_t *payload = NULL;
if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STATUS, timeout_ms,
CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload)) {
CBM_DAEMON_STATUS_RESPONSE_SIZE, &payload,
&status_out->muted_endpoint_holder_pid)) {
return false;
}
status_out->permanent = (payload[1] & 0x01U) != 0U;
@@ -2648,7 +2733,7 @@ bool cbm_daemon_runtime_request_stop(const cbm_daemon_ipc_endpoint_t *endpoint,
memset(result_out, 0, sizeof(*result_out));
uint8_t *payload = NULL;
if (!runtime_control_request_send(endpoint, identity, CBM_DAEMON_RUNTIME_OP_STOP, timeout_ms,
CBM_DAEMON_STOP_RESPONSE_SIZE, &payload)) {
CBM_DAEMON_STOP_RESPONSE_SIZE, &payload, NULL)) {
return false;
}
result_out->accepted = (payload[1] & 0x01U) != 0U;
@@ -2691,6 +2776,13 @@ cbm_daemon_runtime_client_t *cbm_daemon_runtime_client_connect(
frame.length == CBM_DAEMON_RENDEZVOUS_RESPONSE_SIZE &&
runtime_hello_response_decode(payload, result_out);
free(payload);
if (received != 1) {
/* The kernel completed the pipe/socket connect, so a server process
* exists, yet the HELLO went unanswered. Name that holder: a dead
* runtime behind a live endpoint is otherwise indistinguishable from
* absence, and the 2026-08-29 zombie hid behind exactly that gap. */
result_out->muted_endpoint_holder_pid = cbm_daemon_ipc_connection_peer_pid(connection);
}
if (!valid || result_out->status != CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED) {
cbm_daemon_ipc_connection_close(connection);
return NULL;
+20
View File
@@ -214,6 +214,12 @@ typedef struct {
cbm_daemon_client_id_t client_id;
/* Kernel-authenticated PID of this client as observed by the daemon. */
uint64_t authenticated_process_id;
/* Nonzero only when the transport connect succeeded but no valid response
* arrived in time: the kernel-reported PID of the process that holds the
* endpoint. A held-but-mute endpoint is a live process whose runtime is
* wedged (2026-08-29 zombie incident) — callers must report this PID
* rather than fold the failure into plain absence. */
uint64_t muted_endpoint_holder_pid;
cbm_daemon_conflict_t conflict;
char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE];
} cbm_daemon_runtime_connect_result_t;
@@ -265,6 +271,11 @@ typedef struct {
uint32_t client_pids[CBM_DAEMON_CONTROL_CLIENT_CAP];
char build_fingerprint[CBM_DAEMON_BUILD_FINGERPRINT_SIZE];
char semantic_version[CBM_DAEMON_SEMVER_SIZE];
/* Meaningful even when the status request itself FAILED: nonzero when the
* endpoint accepted the transport connect but answered nothing — the
* kernel-reported PID of the mute holder. `daemon status` uses this to
* name a zombie generation instead of reporting bare "not running". */
uint64_t muted_endpoint_holder_pid;
} cbm_daemon_runtime_status_t;
typedef struct {
@@ -421,6 +432,15 @@ void cbm_daemon_runtime_force_peer_image_unverified_for_testing(bool force);
* i.e. the tamper case that must still be rejected after unverifiable images
* became admissible. */
void cbm_daemon_runtime_force_peer_image_mismatch_for_testing(bool force);
/* Abandoned-request containment seams (2026-08-29 zombie incident). The
* timeout override shrinks the join ceiling to test scale; zero restores the
* production constant. The hook replaces the terminal containment stop with a
* recordable callback so an in-process harness survives the trigger; NULL
* restores process termination. */
typedef void (*cbm_daemon_runtime_containment_hook_t)(const char *component);
void cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(uint32_t timeout_ms);
void cbm_daemon_runtime_set_containment_hook_for_testing(
cbm_daemon_runtime_containment_hook_t hook);
#endif
#endif /* CBM_DAEMON_RUNTIME_H */
+5 -1
View File
@@ -463,7 +463,11 @@ cbm_version_cohort_status_t cbm_version_cohort_acquire(cbm_version_cohort_manage
* which a mismatched client can join. Name it: "was the lifetime lock
* held?" is exactly what separates a local run (conflict raised) from
* a CI run (client admitted), and it was not observable in any log. */
cbm_log_info("version_cohort.claimed_unheld", "build",
/* Key wording matters here: the old name "claimed_unheld" read as a
* stale-claim anomaly and derailed the 2026-08-29 zombie diagnosis,
* when the record actually marks the HEALTHY fresh-claim path that
* every normal start logs. */
cbm_log_info("version_cohort.claimed_fresh", "prior_holder", "none", "build",
identity->build_fingerprint ? identity->build_fingerprint : "<null>");
status = version_cohort_claim_new(lease, identity, deadline_ms);
} else if (status == CBM_VERSION_COHORT_BUSY) {
+13
View File
@@ -2258,6 +2258,19 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo
if (strcmp(subcommand, "status") == 0) {
if (!active) {
if (status.muted_endpoint_holder_pid != 0) {
/* Alive process, dead runtime (2026-08-29 zombie class).
* "not running" here sent the operator hunting through
* processes, pipes, and logs by hand; name the holder and the
* recovery instead. */
printf("daemon: not responding (endpoint held by pid %llu)\n",
(unsigned long long)status.muted_endpoint_holder_pid);
printf("hint: that process holds the daemon endpoint but answered nothing; "
"its runtime is likely dead. Terminate pid %llu, then run "
"`codebase-memory-mcp daemon start`.\n",
(unsigned long long)status.muted_endpoint_holder_pid);
return EXIT_FAILURE;
}
printf("daemon: not running\n");
printf("hint: `codebase-memory-mcp daemon start` keeps a daemon warm so CLI "
"commands and hooks skip the per-command startup cost.\n");
+27
View File
@@ -780,6 +780,32 @@ TEST(daemon_bootstrap_rejected_connect_is_reserved_and_never_unavailable) {
PASS();
}
/* 2026-08-29 zombie regression: a connect that reached a live process but got
* no answer names that holder in muted_endpoint_holder_pid. That endpoint is
* owned regardless of what the advisory locks read, so classification must be
* RESERVED on every lifetime answer — never absence, never a spawn license. */
TEST(daemon_bootstrap_mute_endpoint_holder_is_reserved_and_never_unavailable) {
cbm_daemon_runtime_connect_result_t mute = {0};
mute.status = CBM_DAEMON_RUNTIME_CONNECT_ERROR;
mute.muted_endpoint_holder_pid = 4242;
ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&mute, 1),
CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED);
ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&mute, 0),
CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED);
ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&mute, -1),
CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED);
/* A protocol-level rejection still wins over the holder pid: an answering
* generation is more precise evidence than a silent one. */
cbm_daemon_runtime_connect_result_t rejected = {0};
rejected.status = CBM_DAEMON_RUNTIME_CONNECT_REJECTED;
rejected.muted_endpoint_holder_pid = 4242;
snprintf(rejected.message, sizeof(rejected.message), "CBM daemon is stopping");
ASSERT_EQ(cbm_daemon_bootstrap_classify_failed_connect(&rejected, 0),
CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL);
PASS();
}
TEST(daemon_bootstrap_concurrent_first_clients_spawn_one_daemon) {
bootstrap_endpoint_fixture_t fixture;
ASSERT_TRUE(bootstrap_endpoint_fixture_start(&fixture, "startup-race"));
@@ -872,6 +898,7 @@ SUITE(daemon_bootstrap) {
RUN_TEST(daemon_bootstrap_reserved_then_absent_spawns_replacement);
RUN_TEST(daemon_bootstrap_releases_handoff_when_spawned_generation_is_reserved);
RUN_TEST(daemon_bootstrap_rejected_connect_is_reserved_and_never_unavailable);
RUN_TEST(daemon_bootstrap_mute_endpoint_holder_is_reserved_and_never_unavailable);
RUN_TEST(daemon_bootstrap_concurrent_first_clients_spawn_one_daemon);
#ifdef __APPLE__
RUN_TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous);
+222
View File
@@ -3946,6 +3946,224 @@ TEST(daemon_runtime_noncooperative_callback_does_not_detach_or_unbound_stop) {
PASS();
}
#if defined(CBM_ENABLE_TEST_SEAMS)
static atomic_int runtime_test_containment_fired;
static atomic_bool runtime_test_containment_component_exact;
static void runtime_test_containment_hook(const char *component) {
if (component && strcmp(component, "application_request_join") == 0) {
atomic_store_explicit(&runtime_test_containment_component_exact, true,
memory_order_release);
}
atomic_fetch_add_explicit(&runtime_test_containment_fired, 1, memory_order_release);
}
/* 2026-08-29 zombie regression: a disconnecting worker joined its in-flight
* application request with no deadline. A handler that ignored cancellation
* therefore wedged the disconnect path forever and the dead generation kept
* the endpoint. The join must now reach the containment boundary in bounded
* time; the seam hook stands in for the terminal process stop so the harness
* can then supply cooperation and prove a clean join and teardown. */
TEST(daemon_runtime_abandoned_request_join_reaches_containment_in_bounded_time) {
static const uint8_t request[] = {'w', 'e', 'd', 'g', 'e'};
enum { JOIN_BOUND_MS = 150, CONTAINMENT_OBSERVED_MAX_MS = 5000 };
cbm_daemon_build_identity_t identity =
runtime_test_identity("2.4.0", runtime_test_self_build());
runtime_application_context_t context;
runtime_application_context_init(&context, true);
atomic_store_explicit(&context.ignore_first_request_cancel, true, memory_order_release);
atomic_store(&runtime_test_containment_fired, 0);
atomic_store(&runtime_test_containment_component_exact, false);
atomic_bool request_thread_completed;
atomic_init(&request_thread_completed, false);
runtime_test_fixture_t fixture;
bool started = runtime_test_fixture_start_application(&fixture, "application-abandoned-join",
&identity, &context);
cbm_daemon_runtime_connect_result_t result = {0};
cbm_daemon_runtime_client_t *client = NULL;
runtime_application_client_call_t call = {
.request = request,
.request_length = (uint32_t)sizeof(request),
.completed = &request_thread_completed,
.status = CBM_DAEMON_RUNTIME_APPLICATION_OK,
};
cbm_thread_t request_thread;
int request_thread_create_rc = -1;
int request_thread_join_rc = -1;
bool request_thread_started = false;
bool callback_started = false;
bool close_begun = false;
bool containment_observed = false;
uint64_t containment_elapsed_ms = UINT64_MAX;
bool exited_after_release = false;
cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(JOIN_BOUND_MS);
cbm_daemon_runtime_set_containment_hook_for_testing(runtime_test_containment_hook);
if (started) {
client = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity,
RUNTIME_TEST_TIMEOUT_MS, &result);
}
if (client) {
call.client = client;
request_thread_create_rc = cbm_thread_create(
&request_thread, 128U * 1024U, runtime_application_client_request_thread, &call);
request_thread_started = request_thread_create_rc == 0;
callback_started =
request_thread_started &&
runtime_test_wait_atomic_bool(&context.first_request_started, RUNTIME_TEST_TIMEOUT_MS);
}
if (callback_started) {
uint64_t containment_started_ms = cbm_now_ms();
close_begun = cbm_daemon_runtime_client_close_begin(client);
uint64_t deadline = cbm_now_ms() + CONTAINMENT_OBSERVED_MAX_MS;
while (atomic_load_explicit(&runtime_test_containment_fired, memory_order_acquire) == 0 &&
cbm_now_ms() < deadline) {
struct timespec pause = {.tv_sec = 0, .tv_nsec = 1000000};
(void)cbm_nanosleep(&pause, NULL);
}
containment_observed =
atomic_load_explicit(&runtime_test_containment_fired, memory_order_acquire) > 0;
containment_elapsed_ms = cbm_now_ms() - containment_started_ms;
}
/* Cooperation after the boundary: the reap waits for the released handler
* so join and teardown stay provably leak-free under the sanitizers. */
atomic_store_explicit(&context.release_first_request, true, memory_order_release);
if (request_thread_started) {
request_thread_join_rc = cbm_thread_join(&request_thread);
}
if (client) {
(void)cbm_daemon_runtime_client_close_finish(client, RUNTIME_TEST_TIMEOUT_MS);
client = NULL;
}
if (started) {
exited_after_release =
cbm_daemon_runtime_service_wait_exited(fixture.service, RUNTIME_TEST_TIMEOUT_MS);
}
free(call.response);
runtime_test_fixture_finish(&fixture);
cbm_daemon_runtime_set_containment_hook_for_testing(NULL);
cbm_daemon_runtime_set_abandoned_request_join_timeout_for_testing(0);
ASSERT_TRUE(started);
ASSERT_EQ(result.status, CBM_DAEMON_RUNTIME_CONNECT_ACCEPTED);
ASSERT_EQ(request_thread_create_rc, 0);
ASSERT_TRUE(callback_started);
ASSERT_TRUE(close_begun);
ASSERT_TRUE(containment_observed);
ASSERT_TRUE(containment_elapsed_ms <= CONTAINMENT_OBSERVED_MAX_MS);
ASSERT_EQ(atomic_load(&runtime_test_containment_fired), 1);
ASSERT_TRUE(atomic_load(&runtime_test_containment_component_exact));
ASSERT_EQ(request_thread_join_rc, 0);
ASSERT_TRUE(exited_after_release);
ASSERT_EQ(atomic_load(&context.opened), 1);
ASSERT_EQ(atomic_load(&context.requests), 1);
ASSERT_EQ(atomic_load(&context.cancelled), 1);
ASSERT_EQ(atomic_load(&context.closed), 1);
PASS();
}
#endif
typedef struct {
cbm_daemon_ipc_listener_t *listener;
cbm_daemon_ipc_connection_t *held[2];
atomic_int held_count;
atomic_bool stop;
} runtime_mute_holder_t;
/* Accepts up to two transport connections and never answers a frame: the
* kernel-level shape of the 2026-08-29 zombie (endpoint owned, runtime dead). */
static void *runtime_test_mute_holder_thread(void *opaque) {
runtime_mute_holder_t *holder = opaque;
while (!atomic_load_explicit(&holder->stop, memory_order_acquire) &&
atomic_load_explicit(&holder->held_count, memory_order_acquire) < 2) {
cbm_daemon_ipc_connection_t *connection = NULL;
int accepted = cbm_daemon_ipc_accept(holder->listener, 50, &connection);
if (accepted == 1 && connection) {
int index = atomic_load_explicit(&holder->held_count, memory_order_acquire);
holder->held[index] = connection;
atomic_store_explicit(&holder->held_count, index + 1, memory_order_release);
}
}
return NULL;
}
static uint64_t runtime_test_self_process_id(void) {
#ifdef _WIN32
return (uint64_t)GetCurrentProcessId();
#else
return (uint64_t)getpid();
#endif
}
/* 2026-08-29 zombie regression: a held endpoint that answers nothing must
* name its holder. Both the HELLO connect and the one-shot status probe
* fail against a mute holder, and both must report the holder's
* kernel-authenticated pid instead of plain absence. */
TEST(daemon_runtime_mute_endpoint_holder_pid_is_reported) {
enum { MUTE_CONNECT_TIMEOUT_MS = 500 };
cbm_daemon_build_identity_t identity =
runtime_test_identity("2.4.0", runtime_test_self_build());
char parent[RUNTIME_TEST_PATH_CAP] = {0};
char key[CBM_DAEMON_KEY_SIZE] = {0};
char runtime_dir[RUNTIME_TEST_PATH_CAP] = {0};
cbm_daemon_ipc_endpoint_t *endpoint = NULL;
runtime_mute_holder_t holder = {0};
atomic_init(&holder.held_count, 0);
atomic_init(&holder.stop, false);
cbm_thread_t holder_thread;
bool holder_started = false;
bool prepared = th_secure_runtime_parent_new(parent, sizeof(parent), "mute-holder") &&
cbm_daemon_rendezvous_key(key);
endpoint = prepared ? cbm_daemon_ipc_endpoint_new(key, parent) : NULL;
bool runtime_dir_copied =
endpoint &&
runtime_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint));
holder.listener = endpoint ? cbm_daemon_ipc_listen(endpoint) : NULL;
holder_started =
holder.listener &&
cbm_thread_create(&holder_thread, 0, runtime_test_mute_holder_thread, &holder) == 0;
cbm_daemon_runtime_connect_result_t connect_result = {0};
cbm_daemon_runtime_client_t *client = NULL;
cbm_daemon_runtime_status_t status = {0};
bool status_active = true;
if (holder_started) {
client = cbm_daemon_runtime_client_connect(endpoint, &identity, MUTE_CONNECT_TIMEOUT_MS,
&connect_result);
status_active = cbm_daemon_runtime_request_status(endpoint, &identity,
MUTE_CONNECT_TIMEOUT_MS, &status);
}
atomic_store_explicit(&holder.stop, true, memory_order_release);
if (holder_started) {
(void)cbm_thread_join(&holder_thread);
}
for (size_t index = 0; index < 2; index++) {
if (holder.held[index]) {
cbm_daemon_ipc_connection_close(holder.held[index]);
}
}
if (holder.listener) {
cbm_daemon_ipc_listener_close(holder.listener);
}
cbm_daemon_ipc_endpoint_free(endpoint);
(void)cbm_rmdir(runtime_dir);
(void)cbm_rmdir(parent);
ASSERT_TRUE(prepared);
ASSERT_TRUE(runtime_dir_copied);
ASSERT_TRUE(holder_started);
ASSERT_NULL(client);
ASSERT_EQ(connect_result.status, CBM_DAEMON_RUNTIME_CONNECT_ERROR);
ASSERT_EQ(connect_result.muted_endpoint_holder_pid, runtime_test_self_process_id());
ASSERT_FALSE(status_active);
ASSERT_EQ(status.muted_endpoint_holder_pid, runtime_test_self_process_id());
PASS();
}
TEST(daemon_runtime_application_busy_cap_and_malformed_are_isolated) {
static const uint8_t blocking_request[] = {'f', 'i', 'r', 's', 't'};
static const uint8_t busy_request[] = {'b', 'u', 's', 'y'};
@@ -4795,6 +5013,10 @@ SUITE(daemon_runtime) {
RUN_TEST(daemon_runtime_close_begin_releases_admission_with_inflight_request);
RUN_TEST(daemon_runtime_disconnect_cancels_blocked_non_index_child_and_preserves_other_session);
RUN_TEST(daemon_runtime_noncooperative_callback_does_not_detach_or_unbound_stop);
#if defined(CBM_ENABLE_TEST_SEAMS)
RUN_TEST(daemon_runtime_abandoned_request_join_reaches_containment_in_bounded_time);
#endif
RUN_TEST(daemon_runtime_mute_endpoint_holder_pid_is_reported);
RUN_TEST(daemon_runtime_application_busy_cap_and_malformed_are_isolated);
RUN_TEST(daemon_runtime_malformed_and_zero_cancel_close_only_offending_connections);
}