Merge pull request #2113 from DeusData/fix/1828-enospc-daemon-start-diagnostics

fix(daemon): name errno/path on listen failure and fail the client fast instead of a 30s timeout (#1828)
This commit is contained in:
Martin Vogel
2026-09-12 03:59:26 +02:00
committed by GitHub
10 changed files with 996 additions and 20 deletions
+338 -2
View File
@@ -6,13 +6,17 @@
#include "daemon/ipc.h"
#include "daemon/service.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/log.h"
#include "foundation/platform.h"
#include <errno.h>
#include <limits.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#ifdef _WIN32
#ifndef WIN32_LEAN_AND_MEAN
@@ -38,6 +42,14 @@ extern char **environ;
enum {
BOOTSTRAP_RETRY_NS = 1000000,
/* #1828: how often a waiting client re-reads the start-failure log
* (the wait loop itself ticks every millisecond). */
BOOTSTRAP_START_FAILURE_CHECK_MS = 50,
/* A record this many seconds OLDER than the client's own spawn instant is
* still accepted: a sibling client's attempt that just failed is the same
* evidence, and second-granularity clocks need the slack anyway. */
BOOTSTRAP_START_FAILURE_SKEW_S = 2,
BOOTSTRAP_START_FAILURE_LOG_CAP = 65536,
BOOTSTRAP_COORDINATION_CLEANUP_MS = 500,
BOOTSTRAP_PATH_CAP = 4096,
};
@@ -292,6 +304,270 @@ static _Noreturn void bootstrap_cleanup_fail_stop(const char *component) {
#endif
}
static const char BOOTSTRAP_START_FAILURE_LOG_NAME[] = "cbm-daemon.start-failures.log";
bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity) {
if (!out || capacity == 0) {
return false;
}
out[0] = '\0';
const char *cache = cbm_resolve_cache_dir();
if (!cache || !cache[0]) {
return false;
}
int written = snprintf(out, capacity, "%s/logs", cache);
if (written <= 0 || (size_t)written >= capacity) {
out[0] = '\0';
return false;
}
return true;
}
/* One record field: the line format is tab-separated, so a tab or newline in
* a value (a path, in theory) becomes '?' rather than a parser ambiguity. */
static void bootstrap_record_field(char *out, size_t capacity, const char *value) {
size_t used = 0;
for (const char *cursor = value ? value : ""; *cursor && used + 1 < capacity; cursor++) {
unsigned char ch = (unsigned char)*cursor;
out[used++] = ch == '\t' || ch == '\n' || ch == '\r' ? '?' : (char)ch;
}
out[used] = '\0';
}
static uint64_t bootstrap_current_pid(void) {
#ifdef _WIN32
return (uint64_t)GetCurrentProcessId();
#else
return (uint64_t)getpid();
#endif
}
bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
const char *component) {
if (!log_directory || !log_directory[0] || !endpoint || !component || !component[0]) {
return false;
}
const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint);
if (!runtime_dir || !runtime_dir[0]) {
return false;
}
cbm_daemon_ipc_listen_failure_t detail;
if (!cbm_daemon_ipc_listen_failure_detail(&detail)) {
memset(&detail, 0, sizeof(detail));
}
char safe_runtime_dir[BOOTSTRAP_PATH_CAP];
char safe_component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP];
char safe_stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP];
char safe_path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP];
bootstrap_record_field(safe_runtime_dir, sizeof(safe_runtime_dir), runtime_dir);
bootstrap_record_field(safe_component, sizeof(safe_component), component);
bootstrap_record_field(safe_stage, sizeof(safe_stage), detail.stage);
bootstrap_record_field(safe_path, sizeof(safe_path), detail.path);
FILE *stream = cbm_daemon_ipc_private_log_open(log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME,
BOOTSTRAP_START_FAILURE_LOG_CAP);
if (!stream) {
return false;
}
int written =
fprintf(stream, "v1\t%llu\t%llu\t%s\t%s\t%s\t%d\t%s\n", (unsigned long long)time(NULL),
(unsigned long long)bootstrap_current_pid(), safe_runtime_dir, safe_component,
safe_stage, detail.errno_value, safe_path);
bool ok = written > 0 && fflush(stream) == 0;
if (fclose(stream) != 0) {
ok = false;
}
return ok;
}
/* Parse one "v1\tts\tpid\truntime_dir\tcomponent\tstage\terrno\tpath" line
* (line is NUL-terminated, newline already stripped, and is modified). */
static bool bootstrap_start_failure_parse(char *line, char **runtime_dir_out,
cbm_daemon_bootstrap_start_failure_t *out) {
char *fields[8];
size_t count = 0;
char *cursor = line;
while (count < 8) {
fields[count++] = cursor;
char *tab = strchr(cursor, '\t');
if (!tab) {
break;
}
*tab = '\0';
cursor = tab + 1;
}
if (count != 8 || strcmp(fields[0], "v1") != 0) {
return false;
}
memset(out, 0, sizeof(*out));
out->recorded_at_s = strtoull(fields[1], NULL, 10);
out->pid = strtoull(fields[2], NULL, 10);
*runtime_dir_out = fields[3];
(void)snprintf(out->component, sizeof(out->component), "%s", fields[4]);
(void)snprintf(out->stage, sizeof(out->stage), "%s", fields[5]);
out->errno_value = (int)strtol(fields[6], NULL, 10);
(void)snprintf(out->path, sizeof(out->path), "%s", fields[7]);
return true;
}
static FILE *bootstrap_start_failure_open(const char *log_directory, int *status_out) {
char path[BOOTSTRAP_PATH_CAP];
int written =
snprintf(path, sizeof(path), "%s/%s", log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME);
if (written <= 0 || written >= (int)sizeof(path)) {
*status_out = -1;
return NULL;
}
#ifndef _WIN32
struct stat status;
if (lstat(path, &status) != 0) {
*status_out = errno == ENOENT ? 0 : -1;
return NULL;
}
if (!S_ISREG(status.st_mode) || status.st_uid != geteuid() || status.st_nlink != 1) {
*status_out = -1;
return NULL;
}
#endif
/* The directory is the trust boundary (owner-only, no symlinked
* ancestry); the same validation guards cbm-daemon.log itself. */
if (!cbm_daemon_ipc_private_directory_secure(log_directory)) {
*status_out = -1;
return NULL;
}
FILE *file = cbm_fopen(path, "rb");
if (!file) {
*status_out = errno == ENOENT ? 0 : -1;
return NULL;
}
*status_out = 1;
return file;
}
int cbm_daemon_bootstrap_start_failure_read(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure) {
if (!out_failure) {
return -1;
}
memset(out_failure, 0, sizeof(*out_failure));
if (!log_directory || !log_directory[0] || !endpoint) {
return -1;
}
const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint);
if (!runtime_dir || !runtime_dir[0]) {
return -1;
}
int status = 0;
FILE *file = bootstrap_start_failure_open(log_directory, &status);
if (!file) {
return status;
}
char *buffer = malloc(BOOTSTRAP_START_FAILURE_LOG_CAP + 1U);
if (!buffer) {
(void)fclose(file);
return -1;
}
size_t used = fread(buffer, 1, BOOTSTRAP_START_FAILURE_LOG_CAP, file);
bool read_ok = !ferror(file);
(void)fclose(file);
buffer[used] = '\0';
int found = 0;
if (read_ok) {
char *line = buffer;
while (line && *line) {
char *end = strchr(line, '\n');
if (!end) {
break; /* a partial trailing line is a record still being written */
}
*end = '\0';
char *record_runtime_dir = NULL;
cbm_daemon_bootstrap_start_failure_t candidate;
if (bootstrap_start_failure_parse(line, &record_runtime_dir, &candidate) &&
strcmp(record_runtime_dir, runtime_dir) == 0 &&
candidate.recorded_at_s >= not_before_s) {
*out_failure = candidate; /* later lines are newer */
found = 1;
}
line = end + 1;
}
} else {
found = -1;
}
free(buffer);
return found;
}
void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure,
const char *log_directory, char *out,
size_t capacity) {
if (!out || capacity == 0) {
return;
}
if (!failure) {
out[0] = '\0';
return;
}
const char *what = failure->stage[0] ? failure->stage
: failure->component[0] ? failure->component
: "startup";
char where[BOOTSTRAP_PATH_CAP];
if (log_directory && log_directory[0]) {
(void)snprintf(where, sizeof(where), "see %s/cbm-daemon.log", log_directory);
} else {
(void)snprintf(where, sizeof(where), "see the daemon log");
}
if (failure->errno_value != 0 && failure->path[0]) {
(void)snprintf(out, capacity,
"CBM daemon failed to start: %s failed with %s (%s) at %s; %s", what,
cbm_errno_name(failure->errno_value), strerror(failure->errno_value),
failure->path, where);
} else if (failure->errno_value != 0) {
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed with %s (%s); %s",
what, cbm_errno_name(failure->errno_value), strerror(failure->errno_value),
where);
} else if (failure->path[0]) {
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed at %s; %s", what,
failure->path, where);
} else {
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed; %s", what, where);
}
}
/* After this attempt spawned a daemon: did that daemon (or a sibling attempt
* moments earlier) record a start failure? A found record ends the wait with
* its cause -- respawning a deterministically failing daemon until the
* deadline only produced "active or starting" (#1828). */
static bool bootstrap_start_failure_detected(const cbm_daemon_bootstrap_config_t *config,
const cbm_daemon_bootstrap_ops_t *ops,
uint64_t spawn_wall_s, bool force,
uint64_t *next_check_ms,
cbm_daemon_bootstrap_result_t *result_out) {
if (spawn_wall_s == 0 || !ops->start_failure_probe) {
return false;
}
uint64_t now_ms = cbm_now_ms();
if (!force && now_ms < *next_check_ms) {
return false;
}
*next_check_ms = now_ms + BOOTSTRAP_START_FAILURE_CHECK_MS;
uint64_t not_before_s = spawn_wall_s > BOOTSTRAP_START_FAILURE_SKEW_S
? spawn_wall_s - BOOTSTRAP_START_FAILURE_SKEW_S
: 0;
cbm_daemon_bootstrap_start_failure_t failure;
if (ops->start_failure_probe(ops->context, config->endpoint, not_before_s, &failure) != 1) {
return false;
}
char logs[BOOTSTRAP_PATH_CAP];
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) {
logs[0] = '\0';
}
cbm_daemon_bootstrap_start_failure_format(&failure, logs, result_out->message,
sizeof(result_out->message));
return true;
}
static void bootstrap_pause(uint64_t deadline) {
uint64_t now = cbm_now_ms();
if (now >= deadline) {
@@ -458,7 +734,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
bool lock_acquired = false;
bool generation_observed = probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL;
uint64_t spawn_wall_s = 0;
uint64_t next_failure_check_ms = 0;
bool start_failed = false;
while (cbm_now_ms() < deadline) {
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false,
&next_failure_check_ms, result_out)) {
start_failed = true;
break;
}
if (!bootstrap_probe_is_waitable(probe)) {
break;
}
@@ -508,11 +792,20 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
break;
}
/* Never relaunch a daemon whose predecessor already recorded why it
* could not start; the record is written before that daemon lets go
* of its lifetime reservation, so this unconditional read sees it. */
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, true,
&next_failure_check_ms, result_out)) {
start_failed = true;
break;
}
cbm_daemon_bootstrap_launch_spec_t spec;
bool spec_ready =
config->spawn_permanent
? cbm_daemon_bootstrap_launch_spec_init_permanent(config->executable_path, &spec)
: cbm_daemon_bootstrap_launch_spec_init(config->executable_path, &spec);
spawn_wall_s = (uint64_t)time(NULL);
if (!spec_ready || !ops->startup_lock_prepare_handoff(ops->context, startup_lock) ||
!ops->spawn_daemon(ops->context, &spec)) {
probe = CBM_DAEMON_BOOTSTRAP_PROBE_ERROR;
@@ -537,7 +830,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
if (!bootstrap_probe_is_waitable(probe)) {
break;
}
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false,
&next_failure_check_ms, result_out)) {
start_failed = true;
break;
}
} while (cbm_now_ms() < deadline);
if (start_failed) {
break;
}
if (!lock_acquired) {
continue;
}
@@ -555,7 +856,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
}
result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED;
if (muted_holder_pid != 0) {
if (start_failed) {
/* result_out->message already names the recorded cause. */
} else if (muted_holder_pid != 0) {
/* The one diagnostic the 2026-08-29 zombie recovery had to assemble by
* hand from process, pipe, and log correlation: name the pid that
* holds the endpoint without answering, and say what to do with it. */
@@ -773,6 +1076,17 @@ static bool bootstrap_production_unlock(void *context, cbm_daemon_bootstrap_lock
return released;
}
static int bootstrap_production_start_failure_probe(
void *context, const cbm_daemon_ipc_endpoint_t *endpoint, uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure) {
(void)context;
char logs[BOOTSTRAP_PATH_CAP];
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) {
return -1;
}
return cbm_daemon_bootstrap_start_failure_read(logs, endpoint, not_before_s, out_failure);
}
static bool bootstrap_production_handoff(void *context, cbm_daemon_bootstrap_lock_t lock) {
(void)context;
return cbm_daemon_ipc_startup_lock_prepare_handoff((cbm_daemon_ipc_startup_lock_t *)lock);
@@ -1041,10 +1355,26 @@ static void bootstrap_production_diagnostic(void *context, const char *message)
(void)fflush(stderr);
}
#ifdef CBM_ENABLE_TEST_SEAMS
static cbm_daemon_bootstrap_spawn_fn g_bootstrap_spawn_override_for_test;
static void *g_bootstrap_spawn_override_context_for_test;
void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn,
void *context) {
g_bootstrap_spawn_override_context_for_test = context;
g_bootstrap_spawn_override_for_test = spawn;
}
static bool bootstrap_seam_spawn(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec) {
(void)context;
return g_bootstrap_spawn_override_for_test(g_bootstrap_spawn_override_context_for_test, spec);
}
#endif
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
const cbm_daemon_bootstrap_config_t *config, cbm_daemon_bootstrap_result_t *result_out) {
bootstrap_production_context_t context = {0};
const cbm_daemon_bootstrap_ops_t ops = {
cbm_daemon_bootstrap_ops_t ops = {
.context = &context,
.cohort_acquire = bootstrap_production_cohort_acquire,
.cohort_release = bootstrap_production_cohort_release,
@@ -1054,6 +1384,12 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
.startup_lock_release = bootstrap_production_unlock,
.spawn_daemon = bootstrap_production_spawn,
.visible_diagnostic = bootstrap_production_diagnostic,
.start_failure_probe = bootstrap_production_start_failure_probe,
};
#ifdef CBM_ENABLE_TEST_SEAMS
if (g_bootstrap_spawn_override_for_test) {
ops.spawn_daemon = bootstrap_seam_spawn;
}
#endif
return cbm_daemon_bootstrap_execute_with_ops(config, &ops, result_out);
}
+58
View File
@@ -119,6 +119,47 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect
typedef void *cbm_daemon_bootstrap_lock_t;
typedef void *cbm_daemon_bootstrap_cohort_t;
/* Durable daemon start-failure record (#1828). A detached daemon has no
* stderr and no channel back to the client that launched it; when its
* listener publication fails (a full /tmp, for instance) the client used to
* respawn it for the whole startup deadline and then report "active or
* starting". The host now appends one record per failed start to
* <cache>/logs/cbm-daemon.start-failures.log -- the same owner-only log
* directory as cbm-daemon.log, deliberately NOT the runtime directory: that
* one is exactly what a full /tmp cannot write, and a new artifact there
* would entangle stale-generation cleanup. A waiting client reads the newest
* record for its endpoint that is not older than its own spawn and fails
* immediately with the cause. Diagnostic only: policy never reads it. */
enum { CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP = 32 };
typedef struct {
uint64_t recorded_at_s; /* wall clock, seconds */
uint64_t pid;
char component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; /* host component that refused */
char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; /* IPC stage, "" when none */
int errno_value; /* 0 when unknown */
char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; /* "" when none */
} cbm_daemon_bootstrap_start_failure_t;
/* <cache>/logs, the owner-only directory that holds every daemon log. */
bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity);
/* Append one record for this process's most recent listener failure detail
* (cbm_daemon_ipc_listen_failure_detail) under `component`. */
bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
const char *component);
/* Newest record for `endpoint` recorded at or after not_before_s. Returns 1
* and fills out_failure, 0 when none matches, -1 when the log directory or
* file cannot be validated. */
int cbm_daemon_bootstrap_start_failure_read(const char *log_directory,
const cbm_daemon_ipc_endpoint_t *endpoint,
uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure);
/* The client-facing sentence: "CBM daemon failed to start: <stage> failed
* with ENOSPC (No space left on device) at <path>; see <log>". */
void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure,
const char *log_directory, char *out,
size_t capacity);
/* Injectable OS/runtime boundary used by the deterministic unit contract.
* Production callers use cbm_daemon_bootstrap_execute(), whose built-in
* operations delegate to daemon IPC/runtime and write visible diagnostics to
@@ -149,6 +190,12 @@ typedef struct {
bool (*startup_lock_release)(void *context, cbm_daemon_bootstrap_lock_t *lock_io);
bool (*spawn_daemon)(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec);
void (*visible_diagnostic)(void *context, const char *message);
/* Optional. After this attempt spawned a daemon, report a start-failure
* record for the endpoint not older than not_before_s (1 found, 0 none,
* -1 unreadable). A found record ends the wait immediately (#1828). */
int (*start_failure_probe)(void *context, const cbm_daemon_ipc_endpoint_t *endpoint,
uint64_t not_before_s,
cbm_daemon_bootstrap_start_failure_t *out_failure);
} cbm_daemon_bootstrap_ops_t;
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
@@ -156,6 +203,17 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
/* Test seam for the same state machine. All callbacks are synchronous and
* borrowed for the duration of the call. */
#ifdef CBM_ENABLE_TEST_SEAMS
/* Replace ONLY the production spawn while every other production operation
* (cohort, probe, startup lock, handoff) stays real. A test forks a real
* daemon host in place of exec'ing the product binary, so the complete
* client/daemon rendezvous runs in one deterministic process tree (#1828). */
typedef bool (*cbm_daemon_bootstrap_spawn_fn)(void *context,
const cbm_daemon_bootstrap_launch_spec_t *spec);
void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn,
void *context);
#endif
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops,
cbm_daemon_bootstrap_result_t *result_out);
+20 -1
View File
@@ -5,6 +5,7 @@
#include "daemon/host_internal.h"
#include "daemon/application.h"
#include "daemon/bootstrap.h"
#include "daemon/runtime.h"
#include "daemon/project_lock.h"
#include "daemon/version_cohort.h"
@@ -146,6 +147,18 @@ static bool host_log_open(char conflict_log_out[HOST_PATH_CAP]) {
return true;
}
/* #1828: a detached daemon has no stderr; its start failure must reach the
* client that is waiting for it, or that client burns its whole deadline and
* reports the opposite of the truth. */
static void host_start_failure_record(const cbm_daemon_ipc_endpoint_t *endpoint,
const char *component) {
char logs[HOST_PATH_CAP];
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs)) ||
!cbm_daemon_bootstrap_start_failure_record(logs, endpoint, component)) {
cbm_log_error("daemon.start_failure_record_failed", "component", component);
}
}
static void host_log_close(void) {
cbm_log_set_sink(NULL);
if (g_host_log_mutex_initialized) {
@@ -1055,10 +1068,16 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) {
};
cbm_daemon_runtime_service_t *service =
cbm_daemon_runtime_service_start_reserved(&runtime_config, &lifetime_reservation);
if (!service) {
cbm_log_error("daemon.start_failed", "component", "runtime");
/* Recorded while the lifetime reservation is still held: a client
* that watches this generation vanish finds the cause already there
* and never launches a doomed replacement. */
host_start_failure_record(config->endpoint, "runtime");
}
cbm_daemon_ipc_lifetime_reservation_release(lifetime_reservation);
lifetime_reservation = NULL;
if (!service) {
cbm_log_error("daemon.start_failed", "component", "runtime");
host_state_free(&host);
host_log_close();
host_participant_guard_close(&participant_guard);
+151 -17
View File
@@ -9,6 +9,7 @@
#include "foundation/compat_thread.h"
#include "foundation/log.h"
#include "foundation/macos_acl.h"
#include "foundation/platform.h"
#include "foundation/private_file_lock_internal.h"
#include "foundation/sha256.h"
#include "foundation/secure_random.h"
@@ -53,6 +54,60 @@ const char *cbm_daemon_ipc_validation_detail(void) {
return ipc_validation_detail_buffer;
}
static cbm_daemon_ipc_listen_failure_t ipc_listen_failure;
/* The failure recorder and its reset are only reached from the POSIX socket
* listener (all callers live in the `#ifndef _WIN32` block below); the Windows
* named-pipe listener does its own reporting. Guarding them keeps the always-
* compiled struct + accessor cross-platform while avoiding -Werror,
* -Wunused-function on the Windows build (#1828 CI). */
#ifndef _WIN32
static void ipc_listen_failure_reset(void) {
memset(&ipc_listen_failure, 0, sizeof(ipc_listen_failure));
}
/* Record and log one listener failure. `name` is an artifact inside the
* endpoint's runtime directory, "" for the directory itself, or NULL when
* the stage has no path. The log line names the errno symbolically and the
* full path, so `daemon.ipc.listen_failed stage=pending_publication` alone
* (the whole trace #1828's reporter had) can no longer happen. */
static void ipc_listen_failed(const char *runtime_dir, const char *stage, int errno_value,
const char *name) {
ipc_listen_failure_reset();
(void)snprintf(ipc_listen_failure.stage, sizeof(ipc_listen_failure.stage), "%s",
stage ? stage : "");
ipc_listen_failure.errno_value = errno_value;
if (runtime_dir && name) {
int written = name[0] ? snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path),
"%s/%s", runtime_dir, name)
: snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path),
"%s", runtime_dir);
if (written <= 0 || written >= (int)sizeof(ipc_listen_failure.path)) {
ipc_listen_failure.path[0] = '\0';
}
}
const char *errno_name = errno_value != 0 ? cbm_errno_name(errno_value) : NULL;
if (errno_name && ipc_listen_failure.path[0]) {
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name, "path",
ipc_listen_failure.path);
} else if (errno_name) {
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name);
} else if (ipc_listen_failure.path[0]) {
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "path", ipc_listen_failure.path);
} else {
cbm_log_error("daemon.ipc.listen_failed", "stage", stage);
}
}
#endif /* !_WIN32 */
bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out) {
if (!out) {
return false;
}
*out = ipc_listen_failure;
return ipc_listen_failure.stage[0] != '\0';
}
static bool instance_key_valid(const char *key) {
if (!key) {
return false;
@@ -122,6 +177,19 @@ void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned
memory_order_release);
}
#ifndef _WIN32
static atomic_int g_posix_record_write_failure_errno_for_test;
#endif
void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value) {
#ifndef _WIN32
atomic_store_explicit(&g_posix_record_write_failure_errno_for_test, errno_value,
memory_order_release);
#else
(void)errno_value;
#endif
}
#ifdef _WIN32
static cbm_daemon_ipc_startup_gate_fn g_startup_gate_for_test;
static void *g_startup_gate_context_for_test;
@@ -1999,6 +2067,29 @@ static bool posix_fd_write_all(int fd, const uint8_t *buffer, size_t length) {
return true;
}
/* The failing step of the most recent record publication in this process:
* its errno (0 when a validation predicate, not a syscall, refused) and the
* artifact name it was operating on. Read by the listener right after a
* publish returns false. */
static int posix_publish_failure_errno;
static char posix_publish_failure_name[NAME_MAX + 1];
static void posix_publish_failure_note(int errno_value, const char *name) {
posix_publish_failure_errno = errno_value;
(void)snprintf(posix_publish_failure_name, sizeof(posix_publish_failure_name), "%s",
name ? name : "");
}
static bool posix_record_fd_write(int fd, const uint8_t *buffer, size_t length) {
int injected =
atomic_load_explicit(&g_posix_record_write_failure_errno_for_test, memory_order_acquire);
if (injected != 0) {
errno = injected;
return false;
}
return posix_fd_write_all(fd, buffer, length);
}
static bool posix_fd_pread_all(int fd, uint8_t *buffer, size_t length) {
size_t offset = 0;
while (offset < length) {
@@ -2353,50 +2444,75 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin
return false;
}
struct stat existing;
errno = 0;
if (fstatat(endpoint->dir_fd, record_name, &existing, AT_SYMLINK_NOFOLLOW) == 0 ||
errno != ENOENT) {
posix_publish_failure_note(errno == 0 ? EEXIST : errno, record_name);
return false;
}
uint8_t record[POSIX_SOCKET_RECORD_SIZE];
if (!posix_socket_record_encode(magic, source, record)) {
posix_publish_failure_note(0, record_name);
return false;
}
char temp_name[NAME_MAX + 1];
if (!posix_socket_record_temp_name(record_name, temp_name)) {
posix_publish_failure_note(ENAMETOOLONG, record_name);
return false;
}
int fd = openat(endpoint->dir_fd, temp_name,
O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600);
if (fd < 0) {
posix_publish_failure_note(errno, temp_name);
return false;
}
struct stat created = {0};
bool temp_exists = true;
bool stable_linked = false;
errno = 0;
bool ok = fd_set_cloexec(fd) && fchmod(fd, 0600) == 0 &&
private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) &&
posix_fd_write_all(fd, record, sizeof(record)) && fsync(fd) == 0 &&
posix_record_fd_write(fd, record, sizeof(record)) && fsync(fd) == 0 &&
private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) &&
created.st_size == (off_t)POSIX_SOCKET_RECORD_SIZE &&
endpoint_runtime_still_valid(endpoint);
if (!ok) {
posix_publish_failure_note(errno, temp_name);
}
if (ok) {
posix_record_publication_stage_reached(magic, false);
errno = 0;
ok =
posix_linkat_no_follow(endpoint->dir_fd, temp_name, endpoint->dir_fd, record_name) == 0;
stable_linked = ok;
if (!ok) {
posix_publish_failure_note(errno, record_name);
}
}
if (ok) {
posix_record_publication_stage_reached(magic, true);
errno = 0;
ok = posix_path_unlink_regular_if_matches(endpoint->dir_fd, temp_name, created.st_dev,
created.st_ino, 2);
temp_exists = !ok;
if (!ok) {
posix_publish_failure_note(errno, temp_name);
}
}
if (ok) {
errno = 0;
ok = posix_directory_sync(endpoint->dir_fd);
if (!ok) {
posix_publish_failure_note(errno, "");
}
}
errno = 0;
if (close(fd) != 0) {
if (ok) {
posix_publish_failure_note(errno, temp_name);
}
ok = false;
}
@@ -2408,6 +2524,9 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin
posix_socket_identity_equal(&published_record.identity, &source->identity) &&
strcmp(published_record.anchor_name, source->anchor_name) == 0 &&
published_status.st_dev == created.st_dev && published_status.st_ino == created.st_ino;
if (!ok) {
posix_publish_failure_note(0, record_name);
}
}
if (!ok) {
if (stable_linked) {
@@ -2723,8 +2842,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
cbm_daemon_ipc_lifetime_reservation_t **reservation_io) {
cbm_daemon_ipc_lifetime_reservation_t *lifetime_reservation =
reservation_io ? *reservation_io : NULL;
ipc_listen_failure_reset();
posix_publish_failure_note(0, NULL);
const char *runtime_dir = endpoint ? endpoint->runtime_dir : NULL;
if (!lifetime_reservation_matches_endpoint(endpoint, lifetime_reservation)) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "reservation_validation");
ipc_listen_failed(runtime_dir, "reservation_validation", 0, NULL);
return NULL;
}
/* Stale removal happens only under the startup lock, before a daemon host
@@ -2734,7 +2856,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
char pending_temp_name[NAME_MAX + 1];
if (!posix_socket_record_temp_name(endpoint->socket_identity_name, identity_temp_name) ||
!posix_socket_record_temp_name(endpoint->socket_pending_name, pending_temp_name)) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "temp_names");
ipc_listen_failed(runtime_dir, "temp_names", ENAMETOOLONG, endpoint->socket_pending_name);
return NULL;
}
struct stat existing;
@@ -2743,27 +2865,32 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
endpoint->socket_pending_name, identity_temp_name, pending_temp_name,
};
bool namespace_absent = true;
const char *occupied_name = "";
int occupied_errno = 0;
for (size_t index = 0; index < sizeof(required_absent) / sizeof(required_absent[0]); index++) {
errno = 0;
if (fstatat(endpoint->dir_fd, required_absent[index], &existing, AT_SYMLINK_NOFOLLOW) ==
0 ||
errno != ENOENT) {
namespace_absent = false;
occupied_name = required_absent[index];
occupied_errno = errno == 0 ? EEXIST : errno;
break;
}
}
if (!endpoint_runtime_still_valid(endpoint) || !namespace_absent) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "namespace_validation");
ipc_listen_failed(runtime_dir, "namespace_validation", occupied_errno, occupied_name);
return NULL;
}
int fd = local_socket_new();
if (fd < 0) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_creation");
ipc_listen_failed(runtime_dir, "socket_creation", errno, NULL);
return NULL;
}
cbm_daemon_ipc_listener_t *listener = calloc(1, sizeof(*listener));
if (!listener) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_allocation");
ipc_listen_failed(runtime_dir, "listener_allocation", ENOMEM, NULL);
(void)close(fd);
return NULL;
}
@@ -2781,7 +2908,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
if (listener->dir_fd < 0 || !fd_set_cloexec(listener->dir_fd) || !listener->runtime_dir ||
!listener->address || !listener->socket_name || !listener->socket_anchor_name ||
!listener->socket_identity_name || !listener->socket_pending_name) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_initialization");
ipc_listen_failed(runtime_dir, "listener_initialization",
listener->dir_fd < 0 ? errno : ENOMEM, NULL);
if (listener->dir_fd >= 0) {
(void)close(listener->dir_fd);
}
@@ -2799,20 +2927,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
struct sockaddr_un address;
socklen_t address_length;
if (!unix_address_set(&address, endpoint->socket_anchor_address, &address_length)) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_address");
ipc_listen_failed(runtime_dir, "socket_address", ENAMETOOLONG,
endpoint->socket_anchor_name);
cbm_daemon_ipc_listener_close(listener);
return NULL;
}
if (bind(fd, (const struct sockaddr *)&address, address_length) != 0) {
int bind_error = errno;
char error_text[32];
(void)snprintf(error_text, sizeof(error_text), "%d", bind_error);
cbm_log_error("daemon.ipc.listen_failed", "bind_errno", error_text);
ipc_listen_failed(runtime_dir, "socket_bind", errno, endpoint->socket_anchor_name);
cbm_daemon_ipc_listener_close(listener);
return NULL;
}
struct stat bound_status;
errno = 0;
bool bound_path_ok = fstatat(endpoint->dir_fd, endpoint->socket_anchor_name, &bound_status,
AT_SYMLINK_NOFOLLOW) == 0 &&
S_ISSOCK(bound_status.st_mode) && bound_status.st_uid == geteuid();
@@ -2832,11 +2959,13 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
anchor_status.st_ino != bound_status.st_ino ||
!posix_socket_identity_from_stat(&anchor_status, &anchor_identity) ||
!posix_directory_sync(endpoint->dir_fd)) {
int security_errno = errno;
if (bound_path_ok) {
posix_bound_socket_unlink_if_matches(endpoint->dir_fd, endpoint->socket_anchor_name,
bound_status.st_dev, bound_status.st_ino);
}
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_security");
ipc_listen_failed(runtime_dir, "socket_security", security_errno,
endpoint->socket_anchor_name);
cbm_daemon_ipc_listener_close(listener);
return NULL;
}
@@ -2853,16 +2982,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
endpoint, endpoint->socket_pending_name, POSIX_SOCKET_PENDING_MAGIC, &pending,
&pending_status.st_dev, &pending_status.st_ino);
if (!pending_published) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_publication");
ipc_listen_failed(runtime_dir, "pending_publication", posix_publish_failure_errno,
posix_publish_failure_name);
posix_publication_abort(endpoint, &anchor_identity, false, NULL, false, NULL);
cbm_daemon_ipc_listener_close(listener);
return NULL;
}
posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_PENDING_DURABLE);
errno = 0;
bool stable_linked = posix_linkat_no_follow(endpoint->dir_fd, endpoint->socket_anchor_name,
endpoint->dir_fd, endpoint->socket_name) == 0 &&
posix_directory_sync(endpoint->dir_fd);
int stable_errno = stable_linked ? 0 : errno;
posix_socket_identity_t stable_identity = {0};
posix_socket_identity_t committed_identity = {0};
struct stat stable_status = {0};
@@ -2877,7 +3009,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
posix_socket_identity_equal(&stable_identity, &committed_identity) &&
posix_socket_inode_equal(&anchor_identity, &committed_identity);
if (!stable_valid) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "stable_publication");
ipc_listen_failed(runtime_dir, "stable_publication", stable_errno, endpoint->socket_name);
posix_publication_abort(endpoint, &anchor_identity, pending_published, &pending_status,
false, NULL);
cbm_daemon_ipc_listener_close(listener);
@@ -2896,7 +3028,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
endpoint, endpoint->socket_identity_name, POSIX_SOCKET_MARKER_MAGIC, &marker,
&marker_status.st_dev, &marker_status.st_ino);
if (!marker_published) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "marker_publication");
ipc_listen_failed(runtime_dir, "marker_publication", posix_publish_failure_errno,
posix_publish_failure_name);
posix_publication_abort(endpoint, &committed_identity, pending_published, &pending_status,
false, NULL);
cbm_daemon_ipc_listener_close(listener);
@@ -2906,10 +3039,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
listener->identity_inode = marker_status.st_ino;
posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_MARKER_DURABLE);
errno = 0;
if (!posix_path_unlink_regular_if_matches(endpoint->dir_fd, endpoint->socket_pending_name,
pending_status.st_dev, pending_status.st_ino, 1) ||
!posix_directory_sync(endpoint->dir_fd)) {
cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_removal");
ipc_listen_failed(runtime_dir, "pending_removal", errno, endpoint->socket_pending_name);
posix_publication_abort(endpoint, &committed_identity, true, &pending_status,
marker_published, &marker_status);
cbm_daemon_ipc_listener_close(listener);
+19
View File
@@ -91,6 +91,25 @@ bool cbm_daemon_ipc_private_directory_secure(const char *directory_path);
* validation refusal (empty string when none). Diagnostic only — callers
* append it to their error messages; policy decisions never read it. */
const char *cbm_daemon_ipc_validation_detail(void);
/* This process's most recent listener-publication failure: the stage that
* refused, the errno the failing step reported (0 when the step reported
* none), and the artifact path it was operating on ("" when no path applies).
* Every `daemon.ipc.listen_failed` log line carries the same three fields.
* Reset at the start of each listen attempt. Returns false while no failure
* has been recorded. Diagnostic only: the daemon host copies it into its
* durable start-failure record so a waiting client can name the cause instead
* of burning its full startup deadline (#1828). */
enum {
CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP = 32,
CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP = 4096
};
typedef struct {
char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP];
int errno_value;
char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP];
} cbm_daemon_ipc_listen_failure_t;
bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out);
#ifdef CBM_ENABLE_TEST_SEAMS
/* #1537: seed the detail so a test can prove the CLI refusal surfaces it. */
void cbm_daemon_ipc_set_validation_detail_for_testing(const char *detail);
+5
View File
@@ -95,6 +95,11 @@ typedef void (*cbm_daemon_ipc_posix_publication_hook_fn)(
void cbm_daemon_ipc_posix_publication_hook_set_for_test(
cbm_daemon_ipc_posix_publication_hook_fn hook, void *context);
void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned int count);
/* #1828: while non-zero, every POSIX socket-record publication write fails
* with this errno. This is the deterministic stand-in for a full runtime
* filesystem (tmpfs ENOSPC): the socket binds, the record file is created,
* and only its data write is refused -- exactly the reporter's failure shape. */
void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value);
/* Deterministic-interleaving seam: fires on the Windows startup path once the
* startup lock is held, before the rendezvous handoff. A test parks here to
+54
View File
@@ -543,6 +543,60 @@ const char *cbm_app_local_dir(void) {
/* ── Cache directory ────────────────────────── */
const char *cbm_errno_name(int error) {
static const struct {
int value;
const char *name;
} names[] = {
{EACCES, "EACCES"},
{EAGAIN, "EAGAIN"},
{EBUSY, "EBUSY"},
{EEXIST, "EEXIST"},
{EFBIG, "EFBIG"},
{EINTR, "EINTR"},
{EINVAL, "EINVAL"},
{EIO, "EIO"},
{EISDIR, "EISDIR"},
{ELOOP, "ELOOP"},
{EMFILE, "EMFILE"},
{EMLINK, "EMLINK"},
{ENAMETOOLONG, "ENAMETOOLONG"},
{ENFILE, "ENFILE"},
{ENOENT, "ENOENT"},
{ENOMEM, "ENOMEM"},
{ENOSPC, "ENOSPC"},
{ENOTDIR, "ENOTDIR"},
{ENOTEMPTY, "ENOTEMPTY"},
{ENXIO, "ENXIO"},
{EPERM, "EPERM"},
{EROFS, "EROFS"},
{ETXTBSY, "ETXTBSY"},
{EXDEV, "EXDEV"},
{ETIMEDOUT, "ETIMEDOUT"},
{ECONNREFUSED, "ECONNREFUSED"},
{EADDRINUSE, "EADDRINUSE"},
{ENOTSOCK, "ENOTSOCK"},
{EPIPE, "EPIPE"},
#ifdef EDQUOT
{EDQUOT, "EDQUOT"},
#endif
#ifdef EOVERFLOW
{EOVERFLOW, "EOVERFLOW"},
#endif
#ifdef ENOTSUP
{ENOTSUP, "ENOTSUP"},
#endif
};
for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) {
if (names[i].value == error) {
return names[i].name;
}
}
static CBM_TLS char fallback[16];
(void)snprintf(fallback, sizeof(fallback), "%d", error);
return fallback;
}
const char *cbm_resolve_cache_dir(void) {
static CBM_TLS char buf[CBM_SZ_4K];
static const char missing[] = "\x1f"
+7
View File
@@ -94,6 +94,13 @@ uint64_t cbm_now_ns(void);
/* Monotonic millisecond timestamp. */
uint64_t cbm_now_ms(void);
/* Symbolic name for an errno value ("ENOSPC"), or the decimal number when the
* value is not in the portable table. The fallback lives in thread-local
* storage; copy it before the next call on the same thread. Diagnostics only:
* a log line that says `errno=ENOSPC path=...` is a one-line diagnosis where
* `stage=pending_publication` alone cost a reporter hours (#1828). */
const char *cbm_errno_name(int error);
/* ── System info ───────────────────────────────────────────────── */
/* Number of available CPU cores. */
+263
View File
@@ -1,18 +1,30 @@
/* RED contract for early process-role classification. */
#include "test_framework.h"
#include "test_helpers.h"
#include "daemon/bootstrap.h"
#include "daemon/host.h"
#include "daemon/ipc.h"
#include "daemon/ipc_internal.h"
#include "daemon/runtime.h"
#include "daemon/service.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h"
#include "foundation/compat_thread.h"
#include "foundation/platform.h"
#include <errno.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#ifndef _WIN32
#include <signal.h>
#include <sys/wait.h>
#include <unistd.h>
#endif
enum {
BOOTSTRAP_TEST_PATH_CAP = 1024,
@@ -874,6 +886,253 @@ TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous) {
}
#endif
#ifndef _WIN32
enum { BOOTSTRAP_ENOSPC_MAX_CHILDREN = 16, BOOTSTRAP_ENOSPC_LOG_CAP = 65536 };
typedef struct {
char parent[BOOTSTRAP_TEST_PATH_CAP];
cbm_daemon_build_identity_t identity;
pid_t children[BOOTSTRAP_ENOSPC_MAX_CHILDREN];
size_t child_count;
size_t spawn_calls;
} bootstrap_enospc_host_t;
/* The production spawn exec's the product binary; this one forks a REAL daemon
* host (cbm_daemon_host_run, the same entry `--cbm-daemon-internal` reaches)
* whose record publication fails with ENOSPC through the inherited seam. */
static bool bootstrap_enospc_host_spawn(void *opaque,
const cbm_daemon_bootstrap_launch_spec_t *spec) {
bootstrap_enospc_host_t *state = opaque;
if (!spec || !spec->detached) {
return false;
}
state->spawn_calls++;
if (state->child_count >= BOOTSTRAP_ENOSPC_MAX_CHILDREN) {
/* RED-run guard only: a client that keeps respawning a doomed daemon
* for its whole deadline must not fork without bound. */
return true;
}
pid_t child = fork();
if (child < 0) {
return false;
}
if (child == 0) {
cbm_daemon_ipc_endpoint_t *endpoint = cbm_daemon_bootstrap_endpoint_new(state->parent);
atomic_int stop_requested = ATOMIC_VAR_INIT(0);
cbm_daemon_host_config_t config = {
.endpoint = endpoint,
.identity = state->identity,
.executable_path = "/enospc-host-test",
.stop_requested = &stop_requested,
};
int run_result = endpoint ? cbm_daemon_host_run(&config) : 0;
_exit(run_result == -1 ? 0 : 50);
}
state->children[state->child_count++] = child;
return true;
}
static void bootstrap_enospc_reap(bootstrap_enospc_host_t *state, int *nonzero_exits) {
*nonzero_exits = 0;
for (size_t i = 0; i < state->child_count; i++) {
int status = 0;
pid_t waited;
do {
waited = waitpid(state->children[i], &status, WNOHANG);
} while (waited < 0 && errno == EINTR);
if (waited == 0) {
(void)kill(state->children[i], SIGKILL);
do {
waited = waitpid(state->children[i], &status, 0);
} while (waited < 0 && errno == EINTR);
}
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
(*nonzero_exits)++;
}
}
}
static bool bootstrap_read_file(const char *path, char *out, size_t capacity) {
FILE *file = cbm_fopen(path, "rb");
if (!file) {
return false;
}
size_t used = fread(out, 1, capacity - 1, file);
out[used] = '\0';
(void)fclose(file);
return true;
}
/* The record contract behind the fail-fast: a real listener failure in this
* process is recorded and read back with stage, errno, and path; a record
* older than the reader's spawn or for another endpoint is not evidence. */
TEST(daemon_bootstrap_start_failure_record_round_trip) {
bootstrap_endpoint_fixture_t fixture;
bootstrap_endpoint_fixture_t other;
ASSERT_TRUE(bootstrap_endpoint_fixture_start(&fixture, "failure-record"));
char other_parent[BOOTSTRAP_TEST_PATH_CAP];
int other_written =
snprintf(other_parent, sizeof(other_parent), "%s/other-XXXXXX", fixture.parent);
ASSERT(other_written > 0 && other_written < (int)sizeof(other_parent));
ASSERT_TRUE(cbm_mkdtemp(other_parent) != NULL);
memset(&other, 0, sizeof(other));
other.endpoint = cbm_daemon_ipc_endpoint_new("1828000000000002", other_parent);
ASSERT_TRUE(other.endpoint != NULL);
char logs[BOOTSTRAP_TEST_PATH_CAP];
int logs_written = snprintf(logs, sizeof(logs), "%s/logs", fixture.parent);
ASSERT(logs_written > 0 && logs_written < (int)sizeof(logs));
char expected_path[BOOTSTRAP_TEST_PATH_CAP];
int path_written = snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp",
cbm_daemon_ipc_endpoint_address(fixture.endpoint));
ASSERT(path_written > 0 && path_written < (int)sizeof(expected_path));
cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC);
cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(fixture.endpoint);
cbm_daemon_ipc_posix_record_write_failure_set_for_test(0);
ASSERT_TRUE(listener == NULL);
uint64_t now_s = (uint64_t)time(NULL);
bool recorded = cbm_daemon_bootstrap_start_failure_record(logs, fixture.endpoint, "runtime");
cbm_daemon_bootstrap_start_failure_t failure;
int found =
cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s - 2, &failure);
cbm_daemon_bootstrap_start_failure_t stale;
int stale_found =
cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s + 60, &stale);
cbm_daemon_bootstrap_start_failure_t foreign;
int foreign_found =
cbm_daemon_bootstrap_start_failure_read(logs, other.endpoint, now_s - 2, &foreign);
char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE];
cbm_daemon_bootstrap_start_failure_format(&failure, logs, message, sizeof(message));
cbm_daemon_ipc_endpoint_free(other.endpoint);
(void)th_rmtree(fixture.parent);
bootstrap_endpoint_fixture_finish(&fixture);
ASSERT_TRUE(recorded);
ASSERT_EQ(found, 1);
ASSERT_STR_EQ(failure.component, "runtime");
ASSERT_STR_EQ(failure.stage, "pending_publication");
ASSERT_EQ(failure.errno_value, ENOSPC);
ASSERT_STR_EQ(failure.path, expected_path);
ASSERT_TRUE(failure.pid == (uint64_t)getpid());
ASSERT_EQ(stale_found, 0);
ASSERT_EQ(foreign_found, 0);
ASSERT_TRUE(strstr(message, "CBM daemon failed to start: pending_publication failed with "
"ENOSPC (") != NULL);
ASSERT_TRUE(strstr(message, expected_path) != NULL);
ASSERT_TRUE(strstr(message, "/cbm-daemon.log") != NULL);
PASS();
}
/* #1828: a daemon that dies at publication (full /tmp) left every client
* waiting the full 30 s and then reporting "active or starting" -- the
* opposite of the truth. A real host is spawned against a runtime directory
* whose record writes fail with ENOSPC; the client must report "failed to
* start" naming the errno and the path by ending the wait on the recorded
* cause, not by exhausting the deadline. The proof is the surfaced record and
* the single spawn (see the assertions), never a wall-clock measurement. */
TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication) {
const char *old_cache = getenv("CBM_CACHE_DIR");
char *saved_cache = old_cache ? cbm_strdup(old_cache) : NULL;
bool snapshot_ok = !old_cache || saved_cache;
bootstrap_endpoint_fixture_t fixture;
bool fixture_ok = snapshot_ok && bootstrap_endpoint_fixture_start(&fixture, "enospc-host");
char cache[BOOTSTRAP_TEST_PATH_CAP] = {0};
char daemon_log[BOOTSTRAP_TEST_PATH_CAP] = {0};
char expected_path[BOOTSTRAP_TEST_PATH_CAP] = {0};
int cache_written =
fixture_ok ? snprintf(cache, sizeof(cache), "%s/cache", fixture.parent) : -1;
int log_written =
fixture_ok ? snprintf(daemon_log, sizeof(daemon_log), "%s/logs/cbm-daemon.log", cache) : -1;
int path_written = fixture_ok ? snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp",
cbm_daemon_ipc_endpoint_address(fixture.endpoint))
: -1;
bool environment_ready = cache_written > 0 && cache_written < (int)sizeof(cache) &&
log_written > 0 && log_written < (int)sizeof(daemon_log) &&
path_written > 0 && path_written < (int)sizeof(expected_path) &&
cbm_mkdir_p(cache, 0700) && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0;
char self_build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE] = {0};
bool identity_ready = environment_ready && cbm_daemon_runtime_process_build_fingerprint(
(uint64_t)getpid(), self_build);
static bootstrap_enospc_host_t host;
memset(&host, 0, sizeof(host));
(void)snprintf(host.parent, sizeof(host.parent), "%s", fixture_ok ? fixture.parent : "");
host.identity = bootstrap_identity("2.4.0", self_build);
cbm_daemon_bootstrap_config_t config = {
.role = CBM_DAEMON_PROCESS_MCP_CLIENT,
.endpoint = fixture.endpoint,
.identity = &host.identity,
.executable_path = "/enospc-host-test",
.connect_timeout_ms = 200,
.startup_timeout_ms = 30000,
};
cbm_daemon_bootstrap_result_t result;
memset(&result, 0, sizeof(result));
cbm_daemon_bootstrap_status_t status = CBM_DAEMON_BOOTSTRAP_FAILED;
if (identity_ready) {
cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC);
cbm_daemon_bootstrap_spawn_override_set_for_test(bootstrap_enospc_host_spawn, &host);
status = cbm_daemon_bootstrap_execute(&config, &result);
cbm_daemon_bootstrap_spawn_override_set_for_test(NULL, NULL);
cbm_daemon_ipc_posix_record_write_failure_set_for_test(0);
}
int nonzero_exits = 0;
bootstrap_enospc_reap(&host, &nonzero_exits);
static char log[BOOTSTRAP_ENOSPC_LOG_CAP];
log[0] = '\0';
bool log_read = bootstrap_read_file(daemon_log, log, sizeof(log));
const char *listen_failed = strstr(log, "msg=daemon.ipc.listen_failed");
bool daemon_named_cause = listen_failed && strstr(listen_failed, "errno=ENOSPC") != NULL &&
strstr(listen_failed, expected_path) != NULL;
bool message_names_failure = strstr(result.message, "failed to start") != NULL;
bool message_names_errno = strstr(result.message, "ENOSPC") != NULL;
bool message_names_path = strstr(result.message, expected_path) != NULL;
bool stale_wording = strstr(result.message, "active or starting") != NULL;
if (saved_cache) {
(void)cbm_setenv("CBM_CACHE_DIR", saved_cache, 1);
} else {
(void)cbm_unsetenv("CBM_CACHE_DIR");
}
free(saved_cache);
if (fixture_ok) {
(void)th_rmtree(fixture.parent);
bootstrap_endpoint_fixture_finish(&fixture);
}
ASSERT_TRUE(snapshot_ok);
ASSERT_TRUE(fixture_ok);
ASSERT_TRUE(environment_ready);
ASSERT_TRUE(identity_ready);
ASSERT_EQ(status, CBM_DAEMON_BOOTSTRAP_FAILED);
ASSERT_TRUE(result.daemon_spawned);
ASSERT_TRUE(host.child_count >= 1);
ASSERT_TRUE(log_read);
ASSERT_TRUE(daemon_named_cause);
/* Fast-fail is proven by the MECHANISM, never by wall-clock (O9: a gate
* never asserts a transient timing window). The recorded ENOSPC cause is
* surfaced verbatim ("failed to start" + errno + path) and the slow
* "active or starting" timeout wording is absent -- that message is emitted
* ONLY on the fast-fail break (cbm_daemon_bootstrap_start_failure_format),
* never on the 30 s deadline path -- and the client stopped after exactly
* one spawn instead of respawning a doomed daemon until the deadline. Any
* regression to the pre-#1828 30 s hang trips these deterministically. */
ASSERT_FALSE(stale_wording);
ASSERT_TRUE(message_names_failure);
ASSERT_TRUE(message_names_errno);
ASSERT_TRUE(message_names_path);
ASSERT_EQ(host.child_count, 1U);
ASSERT_EQ(nonzero_exits, 0);
bootstrap_endpoint_fixture_finish(&fixture);
PASS();
}
#endif
SUITE(daemon_bootstrap) {
RUN_TEST(daemon_bootstrap_classifies_default_and_ui_as_mcp_clients);
RUN_TEST(daemon_bootstrap_classifies_stateless_commands_without_client);
@@ -903,4 +1162,8 @@ SUITE(daemon_bootstrap) {
#ifdef __APPLE__
RUN_TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous);
#endif
#ifndef _WIN32
RUN_TEST(daemon_bootstrap_start_failure_record_round_trip);
RUN_TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication);
#endif
}
+81
View File
@@ -13,6 +13,7 @@
#include "daemon/ipc_internal.h"
#include "foundation/compat.h"
#include "foundation/compat_thread.h"
#include "foundation/log.h"
#include "foundation/platform.h"
#include "foundation/private_file_lock_internal.h"
#include "foundation/subprocess.h"
@@ -5187,6 +5188,83 @@ TEST(daemon_ipc_posix_single_uid_userns_real_smoke_issue1830) {
#endif /* CBM_ENABLE_TEST_SEAMS */
#endif /* !_WIN32 */
#ifndef _WIN32
enum { IPC_TEST_LOG_CAPTURE_CAP = 16384 };
static char ipc_test_log_capture[IPC_TEST_LOG_CAPTURE_CAP];
static size_t ipc_test_log_capture_used;
static void ipc_test_log_capture_sink(const char *line) {
if (!line) {
return;
}
size_t length = strlen(line);
if (ipc_test_log_capture_used + length + 2 > sizeof(ipc_test_log_capture)) {
return;
}
memcpy(ipc_test_log_capture + ipc_test_log_capture_used, line, length);
ipc_test_log_capture_used += length;
ipc_test_log_capture[ipc_test_log_capture_used++] = '\n';
ipc_test_log_capture[ipc_test_log_capture_used] = '\0';
}
/* #1828: a full /tmp made every daemon start die at pending publication, and
* the only durable trace was `daemon.ipc.listen_failed stage=pending_publication`
* -- no syscall, no errno, no path. The reporter needed hours (and a wrong
* `df` on the wrong mount) to find the cause. The failure line must name the
* errno and the exact artifact path that could not be written. */
TEST(daemon_ipc_listen_failure_names_errno_and_path) {
static const char key[] = "1828000000000001";
char parent[TEST_PATH_CAP] = {0};
char runtime_dir[TEST_PATH_CAP] = {0};
char socket_path[TEST_PATH_CAP] = {0};
char pending_path[TEST_PATH_CAP] = {0};
char expected_path[TEST_PATH_CAP + 16] = {0};
cbm_daemon_ipc_endpoint_t *endpoint = NULL;
cbm_daemon_ipc_listener_t *listener = NULL;
bool parent_ok = ipc_test_parent_new(parent, "enospc-diag");
if (parent_ok) {
endpoint = cbm_daemon_ipc_endpoint_new(key, parent);
}
if (endpoint) {
ipc_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint));
ipc_test_copy_path(socket_path, cbm_daemon_ipc_endpoint_address(endpoint));
}
bool paths_ok = endpoint && ipc_test_socket_pending_path(pending_path, socket_path) &&
snprintf(expected_path, sizeof(expected_path), "path=%s.tmp", pending_path) > 0;
if (paths_ok) {
ipc_test_log_capture_used = 0;
ipc_test_log_capture[0] = '\0';
cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC);
cbm_log_set_sink_ex(ipc_test_log_capture_sink, CBM_LOG_SINK_REPLACE);
listener = cbm_daemon_ipc_listen(endpoint);
cbm_log_set_sink(NULL);
cbm_daemon_ipc_posix_record_write_failure_set_for_test(0);
}
const char *failed = strstr(ipc_test_log_capture, "msg=daemon.ipc.listen_failed");
bool stage_named = failed && strstr(failed, "stage=pending_publication") != NULL;
bool errno_named = failed && strstr(failed, "errno=ENOSPC") != NULL;
bool path_named = failed && strstr(failed, expected_path) != NULL;
struct stat leftover;
bool namespace_clean = paths_ok && lstat(socket_path, &leftover) != 0 && errno == ENOENT &&
lstat(pending_path, &leftover) != 0 && errno == ENOENT;
cbm_daemon_ipc_listener_close(listener);
cbm_daemon_ipc_endpoint_free(endpoint);
th_cleanup(parent_ok ? parent : NULL);
ASSERT_TRUE(parent_ok);
ASSERT_TRUE(paths_ok);
ASSERT_TRUE(listener == NULL);
ASSERT_TRUE(failed != NULL);
ASSERT_TRUE(stage_named);
ASSERT_TRUE(errno_named);
ASSERT_TRUE(path_named);
ASSERT_TRUE(namespace_clean);
PASS();
}
#endif
SUITE(daemon_ipc) {
RUN_TEST(daemon_ipc_pending_timeout_race_returns_completed_io);
RUN_TEST(daemon_ipc_pending_wait_failure_cancels_and_drains);
@@ -5260,5 +5338,8 @@ SUITE(daemon_ipc) {
RUN_TEST(daemon_ipc_posix_private_directory_rejects_world_writable_ancestor);
RUN_TEST(daemon_ipc_posix_private_log_rejects_symlinks_and_is_owner_only);
RUN_TEST(daemon_ipc_posix_rejects_non_socket_and_symlink_endpoints);
#ifndef _WIN32
RUN_TEST(daemon_ipc_listen_failure_names_errno_and_path);
#endif
#endif
}