mirror of
https://github.com/DeusData/codebase-memory-mcp.git
synced 2026-11-04 08:13:37 +08:00
Merge pull request #2113 from DeusData/fix/1828-enospc-daemon-start-diagnostics
fix(daemon): name errno/path on listen failure and fail the client fast instead of a 30s timeout (#1828)
This commit is contained in:
+338
-2
@@ -6,13 +6,17 @@
|
||||
#include "daemon/ipc.h"
|
||||
#include "daemon/service.h"
|
||||
#include "foundation/compat.h"
|
||||
#include "foundation/compat_fs.h"
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/platform.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
@@ -38,6 +42,14 @@ extern char **environ;
|
||||
|
||||
enum {
|
||||
BOOTSTRAP_RETRY_NS = 1000000,
|
||||
/* #1828: how often a waiting client re-reads the start-failure log
|
||||
* (the wait loop itself ticks every millisecond). */
|
||||
BOOTSTRAP_START_FAILURE_CHECK_MS = 50,
|
||||
/* A record this many seconds OLDER than the client's own spawn instant is
|
||||
* still accepted: a sibling client's attempt that just failed is the same
|
||||
* evidence, and second-granularity clocks need the slack anyway. */
|
||||
BOOTSTRAP_START_FAILURE_SKEW_S = 2,
|
||||
BOOTSTRAP_START_FAILURE_LOG_CAP = 65536,
|
||||
BOOTSTRAP_COORDINATION_CLEANUP_MS = 500,
|
||||
BOOTSTRAP_PATH_CAP = 4096,
|
||||
};
|
||||
@@ -292,6 +304,270 @@ static _Noreturn void bootstrap_cleanup_fail_stop(const char *component) {
|
||||
#endif
|
||||
}
|
||||
|
||||
static const char BOOTSTRAP_START_FAILURE_LOG_NAME[] = "cbm-daemon.start-failures.log";
|
||||
|
||||
bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity) {
|
||||
if (!out || capacity == 0) {
|
||||
return false;
|
||||
}
|
||||
out[0] = '\0';
|
||||
const char *cache = cbm_resolve_cache_dir();
|
||||
if (!cache || !cache[0]) {
|
||||
return false;
|
||||
}
|
||||
int written = snprintf(out, capacity, "%s/logs", cache);
|
||||
if (written <= 0 || (size_t)written >= capacity) {
|
||||
out[0] = '\0';
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* One record field: the line format is tab-separated, so a tab or newline in
|
||||
* a value (a path, in theory) becomes '?' rather than a parser ambiguity. */
|
||||
static void bootstrap_record_field(char *out, size_t capacity, const char *value) {
|
||||
size_t used = 0;
|
||||
for (const char *cursor = value ? value : ""; *cursor && used + 1 < capacity; cursor++) {
|
||||
unsigned char ch = (unsigned char)*cursor;
|
||||
out[used++] = ch == '\t' || ch == '\n' || ch == '\r' ? '?' : (char)ch;
|
||||
}
|
||||
out[used] = '\0';
|
||||
}
|
||||
|
||||
static uint64_t bootstrap_current_pid(void) {
|
||||
#ifdef _WIN32
|
||||
return (uint64_t)GetCurrentProcessId();
|
||||
#else
|
||||
return (uint64_t)getpid();
|
||||
#endif
|
||||
}
|
||||
|
||||
bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory,
|
||||
const cbm_daemon_ipc_endpoint_t *endpoint,
|
||||
const char *component) {
|
||||
if (!log_directory || !log_directory[0] || !endpoint || !component || !component[0]) {
|
||||
return false;
|
||||
}
|
||||
const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint);
|
||||
if (!runtime_dir || !runtime_dir[0]) {
|
||||
return false;
|
||||
}
|
||||
cbm_daemon_ipc_listen_failure_t detail;
|
||||
if (!cbm_daemon_ipc_listen_failure_detail(&detail)) {
|
||||
memset(&detail, 0, sizeof(detail));
|
||||
}
|
||||
char safe_runtime_dir[BOOTSTRAP_PATH_CAP];
|
||||
char safe_component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP];
|
||||
char safe_stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP];
|
||||
char safe_path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP];
|
||||
bootstrap_record_field(safe_runtime_dir, sizeof(safe_runtime_dir), runtime_dir);
|
||||
bootstrap_record_field(safe_component, sizeof(safe_component), component);
|
||||
bootstrap_record_field(safe_stage, sizeof(safe_stage), detail.stage);
|
||||
bootstrap_record_field(safe_path, sizeof(safe_path), detail.path);
|
||||
FILE *stream = cbm_daemon_ipc_private_log_open(log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME,
|
||||
BOOTSTRAP_START_FAILURE_LOG_CAP);
|
||||
if (!stream) {
|
||||
return false;
|
||||
}
|
||||
int written =
|
||||
fprintf(stream, "v1\t%llu\t%llu\t%s\t%s\t%s\t%d\t%s\n", (unsigned long long)time(NULL),
|
||||
(unsigned long long)bootstrap_current_pid(), safe_runtime_dir, safe_component,
|
||||
safe_stage, detail.errno_value, safe_path);
|
||||
bool ok = written > 0 && fflush(stream) == 0;
|
||||
if (fclose(stream) != 0) {
|
||||
ok = false;
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Parse one "v1\tts\tpid\truntime_dir\tcomponent\tstage\terrno\tpath" line
|
||||
* (line is NUL-terminated, newline already stripped, and is modified). */
|
||||
static bool bootstrap_start_failure_parse(char *line, char **runtime_dir_out,
|
||||
cbm_daemon_bootstrap_start_failure_t *out) {
|
||||
char *fields[8];
|
||||
size_t count = 0;
|
||||
char *cursor = line;
|
||||
while (count < 8) {
|
||||
fields[count++] = cursor;
|
||||
char *tab = strchr(cursor, '\t');
|
||||
if (!tab) {
|
||||
break;
|
||||
}
|
||||
*tab = '\0';
|
||||
cursor = tab + 1;
|
||||
}
|
||||
if (count != 8 || strcmp(fields[0], "v1") != 0) {
|
||||
return false;
|
||||
}
|
||||
memset(out, 0, sizeof(*out));
|
||||
out->recorded_at_s = strtoull(fields[1], NULL, 10);
|
||||
out->pid = strtoull(fields[2], NULL, 10);
|
||||
*runtime_dir_out = fields[3];
|
||||
(void)snprintf(out->component, sizeof(out->component), "%s", fields[4]);
|
||||
(void)snprintf(out->stage, sizeof(out->stage), "%s", fields[5]);
|
||||
out->errno_value = (int)strtol(fields[6], NULL, 10);
|
||||
(void)snprintf(out->path, sizeof(out->path), "%s", fields[7]);
|
||||
return true;
|
||||
}
|
||||
|
||||
static FILE *bootstrap_start_failure_open(const char *log_directory, int *status_out) {
|
||||
char path[BOOTSTRAP_PATH_CAP];
|
||||
int written =
|
||||
snprintf(path, sizeof(path), "%s/%s", log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME);
|
||||
if (written <= 0 || written >= (int)sizeof(path)) {
|
||||
*status_out = -1;
|
||||
return NULL;
|
||||
}
|
||||
#ifndef _WIN32
|
||||
struct stat status;
|
||||
if (lstat(path, &status) != 0) {
|
||||
*status_out = errno == ENOENT ? 0 : -1;
|
||||
return NULL;
|
||||
}
|
||||
if (!S_ISREG(status.st_mode) || status.st_uid != geteuid() || status.st_nlink != 1) {
|
||||
*status_out = -1;
|
||||
return NULL;
|
||||
}
|
||||
#endif
|
||||
/* The directory is the trust boundary (owner-only, no symlinked
|
||||
* ancestry); the same validation guards cbm-daemon.log itself. */
|
||||
if (!cbm_daemon_ipc_private_directory_secure(log_directory)) {
|
||||
*status_out = -1;
|
||||
return NULL;
|
||||
}
|
||||
FILE *file = cbm_fopen(path, "rb");
|
||||
if (!file) {
|
||||
*status_out = errno == ENOENT ? 0 : -1;
|
||||
return NULL;
|
||||
}
|
||||
*status_out = 1;
|
||||
return file;
|
||||
}
|
||||
|
||||
int cbm_daemon_bootstrap_start_failure_read(const char *log_directory,
|
||||
const cbm_daemon_ipc_endpoint_t *endpoint,
|
||||
uint64_t not_before_s,
|
||||
cbm_daemon_bootstrap_start_failure_t *out_failure) {
|
||||
if (!out_failure) {
|
||||
return -1;
|
||||
}
|
||||
memset(out_failure, 0, sizeof(*out_failure));
|
||||
if (!log_directory || !log_directory[0] || !endpoint) {
|
||||
return -1;
|
||||
}
|
||||
const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint);
|
||||
if (!runtime_dir || !runtime_dir[0]) {
|
||||
return -1;
|
||||
}
|
||||
int status = 0;
|
||||
FILE *file = bootstrap_start_failure_open(log_directory, &status);
|
||||
if (!file) {
|
||||
return status;
|
||||
}
|
||||
char *buffer = malloc(BOOTSTRAP_START_FAILURE_LOG_CAP + 1U);
|
||||
if (!buffer) {
|
||||
(void)fclose(file);
|
||||
return -1;
|
||||
}
|
||||
size_t used = fread(buffer, 1, BOOTSTRAP_START_FAILURE_LOG_CAP, file);
|
||||
bool read_ok = !ferror(file);
|
||||
(void)fclose(file);
|
||||
buffer[used] = '\0';
|
||||
int found = 0;
|
||||
if (read_ok) {
|
||||
char *line = buffer;
|
||||
while (line && *line) {
|
||||
char *end = strchr(line, '\n');
|
||||
if (!end) {
|
||||
break; /* a partial trailing line is a record still being written */
|
||||
}
|
||||
*end = '\0';
|
||||
char *record_runtime_dir = NULL;
|
||||
cbm_daemon_bootstrap_start_failure_t candidate;
|
||||
if (bootstrap_start_failure_parse(line, &record_runtime_dir, &candidate) &&
|
||||
strcmp(record_runtime_dir, runtime_dir) == 0 &&
|
||||
candidate.recorded_at_s >= not_before_s) {
|
||||
*out_failure = candidate; /* later lines are newer */
|
||||
found = 1;
|
||||
}
|
||||
line = end + 1;
|
||||
}
|
||||
} else {
|
||||
found = -1;
|
||||
}
|
||||
free(buffer);
|
||||
return found;
|
||||
}
|
||||
|
||||
void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure,
|
||||
const char *log_directory, char *out,
|
||||
size_t capacity) {
|
||||
if (!out || capacity == 0) {
|
||||
return;
|
||||
}
|
||||
if (!failure) {
|
||||
out[0] = '\0';
|
||||
return;
|
||||
}
|
||||
const char *what = failure->stage[0] ? failure->stage
|
||||
: failure->component[0] ? failure->component
|
||||
: "startup";
|
||||
char where[BOOTSTRAP_PATH_CAP];
|
||||
if (log_directory && log_directory[0]) {
|
||||
(void)snprintf(where, sizeof(where), "see %s/cbm-daemon.log", log_directory);
|
||||
} else {
|
||||
(void)snprintf(where, sizeof(where), "see the daemon log");
|
||||
}
|
||||
if (failure->errno_value != 0 && failure->path[0]) {
|
||||
(void)snprintf(out, capacity,
|
||||
"CBM daemon failed to start: %s failed with %s (%s) at %s; %s", what,
|
||||
cbm_errno_name(failure->errno_value), strerror(failure->errno_value),
|
||||
failure->path, where);
|
||||
} else if (failure->errno_value != 0) {
|
||||
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed with %s (%s); %s",
|
||||
what, cbm_errno_name(failure->errno_value), strerror(failure->errno_value),
|
||||
where);
|
||||
} else if (failure->path[0]) {
|
||||
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed at %s; %s", what,
|
||||
failure->path, where);
|
||||
} else {
|
||||
(void)snprintf(out, capacity, "CBM daemon failed to start: %s failed; %s", what, where);
|
||||
}
|
||||
}
|
||||
|
||||
/* After this attempt spawned a daemon: did that daemon (or a sibling attempt
|
||||
* moments earlier) record a start failure? A found record ends the wait with
|
||||
* its cause -- respawning a deterministically failing daemon until the
|
||||
* deadline only produced "active or starting" (#1828). */
|
||||
static bool bootstrap_start_failure_detected(const cbm_daemon_bootstrap_config_t *config,
|
||||
const cbm_daemon_bootstrap_ops_t *ops,
|
||||
uint64_t spawn_wall_s, bool force,
|
||||
uint64_t *next_check_ms,
|
||||
cbm_daemon_bootstrap_result_t *result_out) {
|
||||
if (spawn_wall_s == 0 || !ops->start_failure_probe) {
|
||||
return false;
|
||||
}
|
||||
uint64_t now_ms = cbm_now_ms();
|
||||
if (!force && now_ms < *next_check_ms) {
|
||||
return false;
|
||||
}
|
||||
*next_check_ms = now_ms + BOOTSTRAP_START_FAILURE_CHECK_MS;
|
||||
uint64_t not_before_s = spawn_wall_s > BOOTSTRAP_START_FAILURE_SKEW_S
|
||||
? spawn_wall_s - BOOTSTRAP_START_FAILURE_SKEW_S
|
||||
: 0;
|
||||
cbm_daemon_bootstrap_start_failure_t failure;
|
||||
if (ops->start_failure_probe(ops->context, config->endpoint, not_before_s, &failure) != 1) {
|
||||
return false;
|
||||
}
|
||||
char logs[BOOTSTRAP_PATH_CAP];
|
||||
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) {
|
||||
logs[0] = '\0';
|
||||
}
|
||||
cbm_daemon_bootstrap_start_failure_format(&failure, logs, result_out->message,
|
||||
sizeof(result_out->message));
|
||||
return true;
|
||||
}
|
||||
|
||||
static void bootstrap_pause(uint64_t deadline) {
|
||||
uint64_t now = cbm_now_ms();
|
||||
if (now >= deadline) {
|
||||
@@ -458,7 +734,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
bool lock_acquired = false;
|
||||
bool generation_observed = probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED ||
|
||||
probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL;
|
||||
uint64_t spawn_wall_s = 0;
|
||||
uint64_t next_failure_check_ms = 0;
|
||||
bool start_failed = false;
|
||||
while (cbm_now_ms() < deadline) {
|
||||
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false,
|
||||
&next_failure_check_ms, result_out)) {
|
||||
start_failed = true;
|
||||
break;
|
||||
}
|
||||
if (!bootstrap_probe_is_waitable(probe)) {
|
||||
break;
|
||||
}
|
||||
@@ -508,11 +792,20 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
break;
|
||||
}
|
||||
|
||||
/* Never relaunch a daemon whose predecessor already recorded why it
|
||||
* could not start; the record is written before that daemon lets go
|
||||
* of its lifetime reservation, so this unconditional read sees it. */
|
||||
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, true,
|
||||
&next_failure_check_ms, result_out)) {
|
||||
start_failed = true;
|
||||
break;
|
||||
}
|
||||
cbm_daemon_bootstrap_launch_spec_t spec;
|
||||
bool spec_ready =
|
||||
config->spawn_permanent
|
||||
? cbm_daemon_bootstrap_launch_spec_init_permanent(config->executable_path, &spec)
|
||||
: cbm_daemon_bootstrap_launch_spec_init(config->executable_path, &spec);
|
||||
spawn_wall_s = (uint64_t)time(NULL);
|
||||
if (!spec_ready || !ops->startup_lock_prepare_handoff(ops->context, startup_lock) ||
|
||||
!ops->spawn_daemon(ops->context, &spec)) {
|
||||
probe = CBM_DAEMON_BOOTSTRAP_PROBE_ERROR;
|
||||
@@ -537,7 +830,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
if (!bootstrap_probe_is_waitable(probe)) {
|
||||
break;
|
||||
}
|
||||
if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false,
|
||||
&next_failure_check_ms, result_out)) {
|
||||
start_failed = true;
|
||||
break;
|
||||
}
|
||||
} while (cbm_now_ms() < deadline);
|
||||
if (start_failed) {
|
||||
break;
|
||||
}
|
||||
if (!lock_acquired) {
|
||||
continue;
|
||||
}
|
||||
@@ -555,7 +856,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
}
|
||||
|
||||
result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED;
|
||||
if (muted_holder_pid != 0) {
|
||||
if (start_failed) {
|
||||
/* result_out->message already names the recorded cause. */
|
||||
} else if (muted_holder_pid != 0) {
|
||||
/* The one diagnostic the 2026-08-29 zombie recovery had to assemble by
|
||||
* hand from process, pipe, and log correlation: name the pid that
|
||||
* holds the endpoint without answering, and say what to do with it. */
|
||||
@@ -773,6 +1076,17 @@ static bool bootstrap_production_unlock(void *context, cbm_daemon_bootstrap_lock
|
||||
return released;
|
||||
}
|
||||
|
||||
static int bootstrap_production_start_failure_probe(
|
||||
void *context, const cbm_daemon_ipc_endpoint_t *endpoint, uint64_t not_before_s,
|
||||
cbm_daemon_bootstrap_start_failure_t *out_failure) {
|
||||
(void)context;
|
||||
char logs[BOOTSTRAP_PATH_CAP];
|
||||
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) {
|
||||
return -1;
|
||||
}
|
||||
return cbm_daemon_bootstrap_start_failure_read(logs, endpoint, not_before_s, out_failure);
|
||||
}
|
||||
|
||||
static bool bootstrap_production_handoff(void *context, cbm_daemon_bootstrap_lock_t lock) {
|
||||
(void)context;
|
||||
return cbm_daemon_ipc_startup_lock_prepare_handoff((cbm_daemon_ipc_startup_lock_t *)lock);
|
||||
@@ -1041,10 +1355,26 @@ static void bootstrap_production_diagnostic(void *context, const char *message)
|
||||
(void)fflush(stderr);
|
||||
}
|
||||
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
static cbm_daemon_bootstrap_spawn_fn g_bootstrap_spawn_override_for_test;
|
||||
static void *g_bootstrap_spawn_override_context_for_test;
|
||||
|
||||
void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn,
|
||||
void *context) {
|
||||
g_bootstrap_spawn_override_context_for_test = context;
|
||||
g_bootstrap_spawn_override_for_test = spawn;
|
||||
}
|
||||
|
||||
static bool bootstrap_seam_spawn(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec) {
|
||||
(void)context;
|
||||
return g_bootstrap_spawn_override_for_test(g_bootstrap_spawn_override_context_for_test, spec);
|
||||
}
|
||||
#endif
|
||||
|
||||
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
|
||||
const cbm_daemon_bootstrap_config_t *config, cbm_daemon_bootstrap_result_t *result_out) {
|
||||
bootstrap_production_context_t context = {0};
|
||||
const cbm_daemon_bootstrap_ops_t ops = {
|
||||
cbm_daemon_bootstrap_ops_t ops = {
|
||||
.context = &context,
|
||||
.cohort_acquire = bootstrap_production_cohort_acquire,
|
||||
.cohort_release = bootstrap_production_cohort_release,
|
||||
@@ -1054,6 +1384,12 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
|
||||
.startup_lock_release = bootstrap_production_unlock,
|
||||
.spawn_daemon = bootstrap_production_spawn,
|
||||
.visible_diagnostic = bootstrap_production_diagnostic,
|
||||
.start_failure_probe = bootstrap_production_start_failure_probe,
|
||||
};
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
if (g_bootstrap_spawn_override_for_test) {
|
||||
ops.spawn_daemon = bootstrap_seam_spawn;
|
||||
}
|
||||
#endif
|
||||
return cbm_daemon_bootstrap_execute_with_ops(config, &ops, result_out);
|
||||
}
|
||||
|
||||
@@ -119,6 +119,47 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect
|
||||
typedef void *cbm_daemon_bootstrap_lock_t;
|
||||
typedef void *cbm_daemon_bootstrap_cohort_t;
|
||||
|
||||
/* Durable daemon start-failure record (#1828). A detached daemon has no
|
||||
* stderr and no channel back to the client that launched it; when its
|
||||
* listener publication fails (a full /tmp, for instance) the client used to
|
||||
* respawn it for the whole startup deadline and then report "active or
|
||||
* starting". The host now appends one record per failed start to
|
||||
* <cache>/logs/cbm-daemon.start-failures.log -- the same owner-only log
|
||||
* directory as cbm-daemon.log, deliberately NOT the runtime directory: that
|
||||
* one is exactly what a full /tmp cannot write, and a new artifact there
|
||||
* would entangle stale-generation cleanup. A waiting client reads the newest
|
||||
* record for its endpoint that is not older than its own spawn and fails
|
||||
* immediately with the cause. Diagnostic only: policy never reads it. */
|
||||
enum { CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP = 32 };
|
||||
typedef struct {
|
||||
uint64_t recorded_at_s; /* wall clock, seconds */
|
||||
uint64_t pid;
|
||||
char component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; /* host component that refused */
|
||||
char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; /* IPC stage, "" when none */
|
||||
int errno_value; /* 0 when unknown */
|
||||
char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; /* "" when none */
|
||||
} cbm_daemon_bootstrap_start_failure_t;
|
||||
|
||||
/* <cache>/logs, the owner-only directory that holds every daemon log. */
|
||||
bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity);
|
||||
/* Append one record for this process's most recent listener failure detail
|
||||
* (cbm_daemon_ipc_listen_failure_detail) under `component`. */
|
||||
bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory,
|
||||
const cbm_daemon_ipc_endpoint_t *endpoint,
|
||||
const char *component);
|
||||
/* Newest record for `endpoint` recorded at or after not_before_s. Returns 1
|
||||
* and fills out_failure, 0 when none matches, -1 when the log directory or
|
||||
* file cannot be validated. */
|
||||
int cbm_daemon_bootstrap_start_failure_read(const char *log_directory,
|
||||
const cbm_daemon_ipc_endpoint_t *endpoint,
|
||||
uint64_t not_before_s,
|
||||
cbm_daemon_bootstrap_start_failure_t *out_failure);
|
||||
/* The client-facing sentence: "CBM daemon failed to start: <stage> failed
|
||||
* with ENOSPC (No space left on device) at <path>; see <log>". */
|
||||
void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure,
|
||||
const char *log_directory, char *out,
|
||||
size_t capacity);
|
||||
|
||||
/* Injectable OS/runtime boundary used by the deterministic unit contract.
|
||||
* Production callers use cbm_daemon_bootstrap_execute(), whose built-in
|
||||
* operations delegate to daemon IPC/runtime and write visible diagnostics to
|
||||
@@ -149,6 +190,12 @@ typedef struct {
|
||||
bool (*startup_lock_release)(void *context, cbm_daemon_bootstrap_lock_t *lock_io);
|
||||
bool (*spawn_daemon)(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec);
|
||||
void (*visible_diagnostic)(void *context, const char *message);
|
||||
/* Optional. After this attempt spawned a daemon, report a start-failure
|
||||
* record for the endpoint not older than not_before_s (1 found, 0 none,
|
||||
* -1 unreadable). A found record ends the wait immediately (#1828). */
|
||||
int (*start_failure_probe)(void *context, const cbm_daemon_ipc_endpoint_t *endpoint,
|
||||
uint64_t not_before_s,
|
||||
cbm_daemon_bootstrap_start_failure_t *out_failure);
|
||||
} cbm_daemon_bootstrap_ops_t;
|
||||
|
||||
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
|
||||
@@ -156,6 +203,17 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute(
|
||||
|
||||
/* Test seam for the same state machine. All callbacks are synchronous and
|
||||
* borrowed for the duration of the call. */
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
/* Replace ONLY the production spawn while every other production operation
|
||||
* (cohort, probe, startup lock, handoff) stays real. A test forks a real
|
||||
* daemon host in place of exec'ing the product binary, so the complete
|
||||
* client/daemon rendezvous runs in one deterministic process tree (#1828). */
|
||||
typedef bool (*cbm_daemon_bootstrap_spawn_fn)(void *context,
|
||||
const cbm_daemon_bootstrap_launch_spec_t *spec);
|
||||
void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn,
|
||||
void *context);
|
||||
#endif
|
||||
|
||||
cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops(
|
||||
const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops,
|
||||
cbm_daemon_bootstrap_result_t *result_out);
|
||||
|
||||
+20
-1
@@ -5,6 +5,7 @@
|
||||
#include "daemon/host_internal.h"
|
||||
|
||||
#include "daemon/application.h"
|
||||
#include "daemon/bootstrap.h"
|
||||
#include "daemon/runtime.h"
|
||||
#include "daemon/project_lock.h"
|
||||
#include "daemon/version_cohort.h"
|
||||
@@ -146,6 +147,18 @@ static bool host_log_open(char conflict_log_out[HOST_PATH_CAP]) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/* #1828: a detached daemon has no stderr; its start failure must reach the
|
||||
* client that is waiting for it, or that client burns its whole deadline and
|
||||
* reports the opposite of the truth. */
|
||||
static void host_start_failure_record(const cbm_daemon_ipc_endpoint_t *endpoint,
|
||||
const char *component) {
|
||||
char logs[HOST_PATH_CAP];
|
||||
if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs)) ||
|
||||
!cbm_daemon_bootstrap_start_failure_record(logs, endpoint, component)) {
|
||||
cbm_log_error("daemon.start_failure_record_failed", "component", component);
|
||||
}
|
||||
}
|
||||
|
||||
static void host_log_close(void) {
|
||||
cbm_log_set_sink(NULL);
|
||||
if (g_host_log_mutex_initialized) {
|
||||
@@ -1055,10 +1068,16 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) {
|
||||
};
|
||||
cbm_daemon_runtime_service_t *service =
|
||||
cbm_daemon_runtime_service_start_reserved(&runtime_config, &lifetime_reservation);
|
||||
if (!service) {
|
||||
cbm_log_error("daemon.start_failed", "component", "runtime");
|
||||
/* Recorded while the lifetime reservation is still held: a client
|
||||
* that watches this generation vanish finds the cause already there
|
||||
* and never launches a doomed replacement. */
|
||||
host_start_failure_record(config->endpoint, "runtime");
|
||||
}
|
||||
cbm_daemon_ipc_lifetime_reservation_release(lifetime_reservation);
|
||||
lifetime_reservation = NULL;
|
||||
if (!service) {
|
||||
cbm_log_error("daemon.start_failed", "component", "runtime");
|
||||
host_state_free(&host);
|
||||
host_log_close();
|
||||
host_participant_guard_close(&participant_guard);
|
||||
|
||||
+151
-17
@@ -9,6 +9,7 @@
|
||||
#include "foundation/compat_thread.h"
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/macos_acl.h"
|
||||
#include "foundation/platform.h"
|
||||
#include "foundation/private_file_lock_internal.h"
|
||||
#include "foundation/sha256.h"
|
||||
#include "foundation/secure_random.h"
|
||||
@@ -53,6 +54,60 @@ const char *cbm_daemon_ipc_validation_detail(void) {
|
||||
return ipc_validation_detail_buffer;
|
||||
}
|
||||
|
||||
static cbm_daemon_ipc_listen_failure_t ipc_listen_failure;
|
||||
|
||||
/* The failure recorder and its reset are only reached from the POSIX socket
|
||||
* listener (all callers live in the `#ifndef _WIN32` block below); the Windows
|
||||
* named-pipe listener does its own reporting. Guarding them keeps the always-
|
||||
* compiled struct + accessor cross-platform while avoiding -Werror,
|
||||
* -Wunused-function on the Windows build (#1828 CI). */
|
||||
#ifndef _WIN32
|
||||
static void ipc_listen_failure_reset(void) {
|
||||
memset(&ipc_listen_failure, 0, sizeof(ipc_listen_failure));
|
||||
}
|
||||
|
||||
/* Record and log one listener failure. `name` is an artifact inside the
|
||||
* endpoint's runtime directory, "" for the directory itself, or NULL when
|
||||
* the stage has no path. The log line names the errno symbolically and the
|
||||
* full path, so `daemon.ipc.listen_failed stage=pending_publication` alone
|
||||
* (the whole trace #1828's reporter had) can no longer happen. */
|
||||
static void ipc_listen_failed(const char *runtime_dir, const char *stage, int errno_value,
|
||||
const char *name) {
|
||||
ipc_listen_failure_reset();
|
||||
(void)snprintf(ipc_listen_failure.stage, sizeof(ipc_listen_failure.stage), "%s",
|
||||
stage ? stage : "");
|
||||
ipc_listen_failure.errno_value = errno_value;
|
||||
if (runtime_dir && name) {
|
||||
int written = name[0] ? snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path),
|
||||
"%s/%s", runtime_dir, name)
|
||||
: snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path),
|
||||
"%s", runtime_dir);
|
||||
if (written <= 0 || written >= (int)sizeof(ipc_listen_failure.path)) {
|
||||
ipc_listen_failure.path[0] = '\0';
|
||||
}
|
||||
}
|
||||
const char *errno_name = errno_value != 0 ? cbm_errno_name(errno_value) : NULL;
|
||||
if (errno_name && ipc_listen_failure.path[0]) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name, "path",
|
||||
ipc_listen_failure.path);
|
||||
} else if (errno_name) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name);
|
||||
} else if (ipc_listen_failure.path[0]) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "path", ipc_listen_failure.path);
|
||||
} else {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", stage);
|
||||
}
|
||||
}
|
||||
#endif /* !_WIN32 */
|
||||
|
||||
bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out) {
|
||||
if (!out) {
|
||||
return false;
|
||||
}
|
||||
*out = ipc_listen_failure;
|
||||
return ipc_listen_failure.stage[0] != '\0';
|
||||
}
|
||||
|
||||
static bool instance_key_valid(const char *key) {
|
||||
if (!key) {
|
||||
return false;
|
||||
@@ -122,6 +177,19 @@ void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned
|
||||
memory_order_release);
|
||||
}
|
||||
|
||||
#ifndef _WIN32
|
||||
static atomic_int g_posix_record_write_failure_errno_for_test;
|
||||
#endif
|
||||
|
||||
void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value) {
|
||||
#ifndef _WIN32
|
||||
atomic_store_explicit(&g_posix_record_write_failure_errno_for_test, errno_value,
|
||||
memory_order_release);
|
||||
#else
|
||||
(void)errno_value;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
static cbm_daemon_ipc_startup_gate_fn g_startup_gate_for_test;
|
||||
static void *g_startup_gate_context_for_test;
|
||||
@@ -1999,6 +2067,29 @@ static bool posix_fd_write_all(int fd, const uint8_t *buffer, size_t length) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/* The failing step of the most recent record publication in this process:
|
||||
* its errno (0 when a validation predicate, not a syscall, refused) and the
|
||||
* artifact name it was operating on. Read by the listener right after a
|
||||
* publish returns false. */
|
||||
static int posix_publish_failure_errno;
|
||||
static char posix_publish_failure_name[NAME_MAX + 1];
|
||||
|
||||
static void posix_publish_failure_note(int errno_value, const char *name) {
|
||||
posix_publish_failure_errno = errno_value;
|
||||
(void)snprintf(posix_publish_failure_name, sizeof(posix_publish_failure_name), "%s",
|
||||
name ? name : "");
|
||||
}
|
||||
|
||||
static bool posix_record_fd_write(int fd, const uint8_t *buffer, size_t length) {
|
||||
int injected =
|
||||
atomic_load_explicit(&g_posix_record_write_failure_errno_for_test, memory_order_acquire);
|
||||
if (injected != 0) {
|
||||
errno = injected;
|
||||
return false;
|
||||
}
|
||||
return posix_fd_write_all(fd, buffer, length);
|
||||
}
|
||||
|
||||
static bool posix_fd_pread_all(int fd, uint8_t *buffer, size_t length) {
|
||||
size_t offset = 0;
|
||||
while (offset < length) {
|
||||
@@ -2353,50 +2444,75 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin
|
||||
return false;
|
||||
}
|
||||
struct stat existing;
|
||||
errno = 0;
|
||||
if (fstatat(endpoint->dir_fd, record_name, &existing, AT_SYMLINK_NOFOLLOW) == 0 ||
|
||||
errno != ENOENT) {
|
||||
posix_publish_failure_note(errno == 0 ? EEXIST : errno, record_name);
|
||||
return false;
|
||||
}
|
||||
uint8_t record[POSIX_SOCKET_RECORD_SIZE];
|
||||
if (!posix_socket_record_encode(magic, source, record)) {
|
||||
posix_publish_failure_note(0, record_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
char temp_name[NAME_MAX + 1];
|
||||
if (!posix_socket_record_temp_name(record_name, temp_name)) {
|
||||
posix_publish_failure_note(ENAMETOOLONG, record_name);
|
||||
return false;
|
||||
}
|
||||
int fd = openat(endpoint->dir_fd, temp_name,
|
||||
O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600);
|
||||
if (fd < 0) {
|
||||
posix_publish_failure_note(errno, temp_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
struct stat created = {0};
|
||||
bool temp_exists = true;
|
||||
bool stable_linked = false;
|
||||
errno = 0;
|
||||
bool ok = fd_set_cloexec(fd) && fchmod(fd, 0600) == 0 &&
|
||||
private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) &&
|
||||
posix_fd_write_all(fd, record, sizeof(record)) && fsync(fd) == 0 &&
|
||||
posix_record_fd_write(fd, record, sizeof(record)) && fsync(fd) == 0 &&
|
||||
private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) &&
|
||||
created.st_size == (off_t)POSIX_SOCKET_RECORD_SIZE &&
|
||||
endpoint_runtime_still_valid(endpoint);
|
||||
if (!ok) {
|
||||
posix_publish_failure_note(errno, temp_name);
|
||||
}
|
||||
if (ok) {
|
||||
posix_record_publication_stage_reached(magic, false);
|
||||
errno = 0;
|
||||
ok =
|
||||
posix_linkat_no_follow(endpoint->dir_fd, temp_name, endpoint->dir_fd, record_name) == 0;
|
||||
stable_linked = ok;
|
||||
if (!ok) {
|
||||
posix_publish_failure_note(errno, record_name);
|
||||
}
|
||||
}
|
||||
if (ok) {
|
||||
posix_record_publication_stage_reached(magic, true);
|
||||
errno = 0;
|
||||
ok = posix_path_unlink_regular_if_matches(endpoint->dir_fd, temp_name, created.st_dev,
|
||||
created.st_ino, 2);
|
||||
temp_exists = !ok;
|
||||
if (!ok) {
|
||||
posix_publish_failure_note(errno, temp_name);
|
||||
}
|
||||
}
|
||||
if (ok) {
|
||||
errno = 0;
|
||||
ok = posix_directory_sync(endpoint->dir_fd);
|
||||
if (!ok) {
|
||||
posix_publish_failure_note(errno, "");
|
||||
}
|
||||
}
|
||||
errno = 0;
|
||||
if (close(fd) != 0) {
|
||||
if (ok) {
|
||||
posix_publish_failure_note(errno, temp_name);
|
||||
}
|
||||
ok = false;
|
||||
}
|
||||
|
||||
@@ -2408,6 +2524,9 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin
|
||||
posix_socket_identity_equal(&published_record.identity, &source->identity) &&
|
||||
strcmp(published_record.anchor_name, source->anchor_name) == 0 &&
|
||||
published_status.st_dev == created.st_dev && published_status.st_ino == created.st_ino;
|
||||
if (!ok) {
|
||||
posix_publish_failure_note(0, record_name);
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
if (stable_linked) {
|
||||
@@ -2723,8 +2842,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
cbm_daemon_ipc_lifetime_reservation_t **reservation_io) {
|
||||
cbm_daemon_ipc_lifetime_reservation_t *lifetime_reservation =
|
||||
reservation_io ? *reservation_io : NULL;
|
||||
ipc_listen_failure_reset();
|
||||
posix_publish_failure_note(0, NULL);
|
||||
const char *runtime_dir = endpoint ? endpoint->runtime_dir : NULL;
|
||||
if (!lifetime_reservation_matches_endpoint(endpoint, lifetime_reservation)) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "reservation_validation");
|
||||
ipc_listen_failed(runtime_dir, "reservation_validation", 0, NULL);
|
||||
return NULL;
|
||||
}
|
||||
/* Stale removal happens only under the startup lock, before a daemon host
|
||||
@@ -2734,7 +2856,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
char pending_temp_name[NAME_MAX + 1];
|
||||
if (!posix_socket_record_temp_name(endpoint->socket_identity_name, identity_temp_name) ||
|
||||
!posix_socket_record_temp_name(endpoint->socket_pending_name, pending_temp_name)) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "temp_names");
|
||||
ipc_listen_failed(runtime_dir, "temp_names", ENAMETOOLONG, endpoint->socket_pending_name);
|
||||
return NULL;
|
||||
}
|
||||
struct stat existing;
|
||||
@@ -2743,27 +2865,32 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
endpoint->socket_pending_name, identity_temp_name, pending_temp_name,
|
||||
};
|
||||
bool namespace_absent = true;
|
||||
const char *occupied_name = "";
|
||||
int occupied_errno = 0;
|
||||
for (size_t index = 0; index < sizeof(required_absent) / sizeof(required_absent[0]); index++) {
|
||||
errno = 0;
|
||||
if (fstatat(endpoint->dir_fd, required_absent[index], &existing, AT_SYMLINK_NOFOLLOW) ==
|
||||
0 ||
|
||||
errno != ENOENT) {
|
||||
namespace_absent = false;
|
||||
occupied_name = required_absent[index];
|
||||
occupied_errno = errno == 0 ? EEXIST : errno;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!endpoint_runtime_still_valid(endpoint) || !namespace_absent) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "namespace_validation");
|
||||
ipc_listen_failed(runtime_dir, "namespace_validation", occupied_errno, occupied_name);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int fd = local_socket_new();
|
||||
if (fd < 0) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_creation");
|
||||
ipc_listen_failed(runtime_dir, "socket_creation", errno, NULL);
|
||||
return NULL;
|
||||
}
|
||||
cbm_daemon_ipc_listener_t *listener = calloc(1, sizeof(*listener));
|
||||
if (!listener) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_allocation");
|
||||
ipc_listen_failed(runtime_dir, "listener_allocation", ENOMEM, NULL);
|
||||
(void)close(fd);
|
||||
return NULL;
|
||||
}
|
||||
@@ -2781,7 +2908,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
if (listener->dir_fd < 0 || !fd_set_cloexec(listener->dir_fd) || !listener->runtime_dir ||
|
||||
!listener->address || !listener->socket_name || !listener->socket_anchor_name ||
|
||||
!listener->socket_identity_name || !listener->socket_pending_name) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_initialization");
|
||||
ipc_listen_failed(runtime_dir, "listener_initialization",
|
||||
listener->dir_fd < 0 ? errno : ENOMEM, NULL);
|
||||
if (listener->dir_fd >= 0) {
|
||||
(void)close(listener->dir_fd);
|
||||
}
|
||||
@@ -2799,20 +2927,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
struct sockaddr_un address;
|
||||
socklen_t address_length;
|
||||
if (!unix_address_set(&address, endpoint->socket_anchor_address, &address_length)) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_address");
|
||||
ipc_listen_failed(runtime_dir, "socket_address", ENAMETOOLONG,
|
||||
endpoint->socket_anchor_name);
|
||||
cbm_daemon_ipc_listener_close(listener);
|
||||
return NULL;
|
||||
}
|
||||
if (bind(fd, (const struct sockaddr *)&address, address_length) != 0) {
|
||||
int bind_error = errno;
|
||||
char error_text[32];
|
||||
(void)snprintf(error_text, sizeof(error_text), "%d", bind_error);
|
||||
cbm_log_error("daemon.ipc.listen_failed", "bind_errno", error_text);
|
||||
ipc_listen_failed(runtime_dir, "socket_bind", errno, endpoint->socket_anchor_name);
|
||||
cbm_daemon_ipc_listener_close(listener);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
struct stat bound_status;
|
||||
errno = 0;
|
||||
bool bound_path_ok = fstatat(endpoint->dir_fd, endpoint->socket_anchor_name, &bound_status,
|
||||
AT_SYMLINK_NOFOLLOW) == 0 &&
|
||||
S_ISSOCK(bound_status.st_mode) && bound_status.st_uid == geteuid();
|
||||
@@ -2832,11 +2959,13 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
anchor_status.st_ino != bound_status.st_ino ||
|
||||
!posix_socket_identity_from_stat(&anchor_status, &anchor_identity) ||
|
||||
!posix_directory_sync(endpoint->dir_fd)) {
|
||||
int security_errno = errno;
|
||||
if (bound_path_ok) {
|
||||
posix_bound_socket_unlink_if_matches(endpoint->dir_fd, endpoint->socket_anchor_name,
|
||||
bound_status.st_dev, bound_status.st_ino);
|
||||
}
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_security");
|
||||
ipc_listen_failed(runtime_dir, "socket_security", security_errno,
|
||||
endpoint->socket_anchor_name);
|
||||
cbm_daemon_ipc_listener_close(listener);
|
||||
return NULL;
|
||||
}
|
||||
@@ -2853,16 +2982,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
endpoint, endpoint->socket_pending_name, POSIX_SOCKET_PENDING_MAGIC, &pending,
|
||||
&pending_status.st_dev, &pending_status.st_ino);
|
||||
if (!pending_published) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_publication");
|
||||
ipc_listen_failed(runtime_dir, "pending_publication", posix_publish_failure_errno,
|
||||
posix_publish_failure_name);
|
||||
posix_publication_abort(endpoint, &anchor_identity, false, NULL, false, NULL);
|
||||
cbm_daemon_ipc_listener_close(listener);
|
||||
return NULL;
|
||||
}
|
||||
posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_PENDING_DURABLE);
|
||||
|
||||
errno = 0;
|
||||
bool stable_linked = posix_linkat_no_follow(endpoint->dir_fd, endpoint->socket_anchor_name,
|
||||
endpoint->dir_fd, endpoint->socket_name) == 0 &&
|
||||
posix_directory_sync(endpoint->dir_fd);
|
||||
int stable_errno = stable_linked ? 0 : errno;
|
||||
posix_socket_identity_t stable_identity = {0};
|
||||
posix_socket_identity_t committed_identity = {0};
|
||||
struct stat stable_status = {0};
|
||||
@@ -2877,7 +3009,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
posix_socket_identity_equal(&stable_identity, &committed_identity) &&
|
||||
posix_socket_inode_equal(&anchor_identity, &committed_identity);
|
||||
if (!stable_valid) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "stable_publication");
|
||||
ipc_listen_failed(runtime_dir, "stable_publication", stable_errno, endpoint->socket_name);
|
||||
posix_publication_abort(endpoint, &anchor_identity, pending_published, &pending_status,
|
||||
false, NULL);
|
||||
cbm_daemon_ipc_listener_close(listener);
|
||||
@@ -2896,7 +3028,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
endpoint, endpoint->socket_identity_name, POSIX_SOCKET_MARKER_MAGIC, &marker,
|
||||
&marker_status.st_dev, &marker_status.st_ino);
|
||||
if (!marker_published) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "marker_publication");
|
||||
ipc_listen_failed(runtime_dir, "marker_publication", posix_publish_failure_errno,
|
||||
posix_publish_failure_name);
|
||||
posix_publication_abort(endpoint, &committed_identity, pending_published, &pending_status,
|
||||
false, NULL);
|
||||
cbm_daemon_ipc_listener_close(listener);
|
||||
@@ -2906,10 +3039,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved(
|
||||
listener->identity_inode = marker_status.st_ino;
|
||||
posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_MARKER_DURABLE);
|
||||
|
||||
errno = 0;
|
||||
if (!posix_path_unlink_regular_if_matches(endpoint->dir_fd, endpoint->socket_pending_name,
|
||||
pending_status.st_dev, pending_status.st_ino, 1) ||
|
||||
!posix_directory_sync(endpoint->dir_fd)) {
|
||||
cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_removal");
|
||||
ipc_listen_failed(runtime_dir, "pending_removal", errno, endpoint->socket_pending_name);
|
||||
posix_publication_abort(endpoint, &committed_identity, true, &pending_status,
|
||||
marker_published, &marker_status);
|
||||
cbm_daemon_ipc_listener_close(listener);
|
||||
|
||||
@@ -91,6 +91,25 @@ bool cbm_daemon_ipc_private_directory_secure(const char *directory_path);
|
||||
* validation refusal (empty string when none). Diagnostic only — callers
|
||||
* append it to their error messages; policy decisions never read it. */
|
||||
const char *cbm_daemon_ipc_validation_detail(void);
|
||||
|
||||
/* This process's most recent listener-publication failure: the stage that
|
||||
* refused, the errno the failing step reported (0 when the step reported
|
||||
* none), and the artifact path it was operating on ("" when no path applies).
|
||||
* Every `daemon.ipc.listen_failed` log line carries the same three fields.
|
||||
* Reset at the start of each listen attempt. Returns false while no failure
|
||||
* has been recorded. Diagnostic only: the daemon host copies it into its
|
||||
* durable start-failure record so a waiting client can name the cause instead
|
||||
* of burning its full startup deadline (#1828). */
|
||||
enum {
|
||||
CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP = 32,
|
||||
CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP = 4096
|
||||
};
|
||||
typedef struct {
|
||||
char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP];
|
||||
int errno_value;
|
||||
char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP];
|
||||
} cbm_daemon_ipc_listen_failure_t;
|
||||
bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out);
|
||||
#ifdef CBM_ENABLE_TEST_SEAMS
|
||||
/* #1537: seed the detail so a test can prove the CLI refusal surfaces it. */
|
||||
void cbm_daemon_ipc_set_validation_detail_for_testing(const char *detail);
|
||||
|
||||
@@ -95,6 +95,11 @@ typedef void (*cbm_daemon_ipc_posix_publication_hook_fn)(
|
||||
void cbm_daemon_ipc_posix_publication_hook_set_for_test(
|
||||
cbm_daemon_ipc_posix_publication_hook_fn hook, void *context);
|
||||
void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned int count);
|
||||
/* #1828: while non-zero, every POSIX socket-record publication write fails
|
||||
* with this errno. This is the deterministic stand-in for a full runtime
|
||||
* filesystem (tmpfs ENOSPC): the socket binds, the record file is created,
|
||||
* and only its data write is refused -- exactly the reporter's failure shape. */
|
||||
void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value);
|
||||
|
||||
/* Deterministic-interleaving seam: fires on the Windows startup path once the
|
||||
* startup lock is held, before the rendezvous handoff. A test parks here to
|
||||
|
||||
@@ -543,6 +543,60 @@ const char *cbm_app_local_dir(void) {
|
||||
|
||||
/* ── Cache directory ────────────────────────── */
|
||||
|
||||
const char *cbm_errno_name(int error) {
|
||||
static const struct {
|
||||
int value;
|
||||
const char *name;
|
||||
} names[] = {
|
||||
{EACCES, "EACCES"},
|
||||
{EAGAIN, "EAGAIN"},
|
||||
{EBUSY, "EBUSY"},
|
||||
{EEXIST, "EEXIST"},
|
||||
{EFBIG, "EFBIG"},
|
||||
{EINTR, "EINTR"},
|
||||
{EINVAL, "EINVAL"},
|
||||
{EIO, "EIO"},
|
||||
{EISDIR, "EISDIR"},
|
||||
{ELOOP, "ELOOP"},
|
||||
{EMFILE, "EMFILE"},
|
||||
{EMLINK, "EMLINK"},
|
||||
{ENAMETOOLONG, "ENAMETOOLONG"},
|
||||
{ENFILE, "ENFILE"},
|
||||
{ENOENT, "ENOENT"},
|
||||
{ENOMEM, "ENOMEM"},
|
||||
{ENOSPC, "ENOSPC"},
|
||||
{ENOTDIR, "ENOTDIR"},
|
||||
{ENOTEMPTY, "ENOTEMPTY"},
|
||||
{ENXIO, "ENXIO"},
|
||||
{EPERM, "EPERM"},
|
||||
{EROFS, "EROFS"},
|
||||
{ETXTBSY, "ETXTBSY"},
|
||||
{EXDEV, "EXDEV"},
|
||||
{ETIMEDOUT, "ETIMEDOUT"},
|
||||
{ECONNREFUSED, "ECONNREFUSED"},
|
||||
{EADDRINUSE, "EADDRINUSE"},
|
||||
{ENOTSOCK, "ENOTSOCK"},
|
||||
{EPIPE, "EPIPE"},
|
||||
#ifdef EDQUOT
|
||||
{EDQUOT, "EDQUOT"},
|
||||
#endif
|
||||
#ifdef EOVERFLOW
|
||||
{EOVERFLOW, "EOVERFLOW"},
|
||||
#endif
|
||||
#ifdef ENOTSUP
|
||||
{ENOTSUP, "ENOTSUP"},
|
||||
#endif
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) {
|
||||
if (names[i].value == error) {
|
||||
return names[i].name;
|
||||
}
|
||||
}
|
||||
static CBM_TLS char fallback[16];
|
||||
(void)snprintf(fallback, sizeof(fallback), "%d", error);
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const char *cbm_resolve_cache_dir(void) {
|
||||
static CBM_TLS char buf[CBM_SZ_4K];
|
||||
static const char missing[] = "\x1f"
|
||||
|
||||
@@ -94,6 +94,13 @@ uint64_t cbm_now_ns(void);
|
||||
/* Monotonic millisecond timestamp. */
|
||||
uint64_t cbm_now_ms(void);
|
||||
|
||||
/* Symbolic name for an errno value ("ENOSPC"), or the decimal number when the
|
||||
* value is not in the portable table. The fallback lives in thread-local
|
||||
* storage; copy it before the next call on the same thread. Diagnostics only:
|
||||
* a log line that says `errno=ENOSPC path=...` is a one-line diagnosis where
|
||||
* `stage=pending_publication` alone cost a reporter hours (#1828). */
|
||||
const char *cbm_errno_name(int error);
|
||||
|
||||
/* ── System info ───────────────────────────────────────────────── */
|
||||
|
||||
/* Number of available CPU cores. */
|
||||
|
||||
@@ -1,18 +1,30 @@
|
||||
/* RED contract for early process-role classification. */
|
||||
#include "test_framework.h"
|
||||
#include "test_helpers.h"
|
||||
|
||||
#include "daemon/bootstrap.h"
|
||||
#include "daemon/host.h"
|
||||
#include "daemon/ipc.h"
|
||||
#include "daemon/ipc_internal.h"
|
||||
#include "daemon/runtime.h"
|
||||
#include "daemon/service.h"
|
||||
#include "foundation/compat.h"
|
||||
#include "foundation/compat_fs.h"
|
||||
#include "foundation/compat_thread.h"
|
||||
#include "foundation/platform.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#ifndef _WIN32
|
||||
#include <signal.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
enum {
|
||||
BOOTSTRAP_TEST_PATH_CAP = 1024,
|
||||
@@ -874,6 +886,253 @@ TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous) {
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef _WIN32
|
||||
enum { BOOTSTRAP_ENOSPC_MAX_CHILDREN = 16, BOOTSTRAP_ENOSPC_LOG_CAP = 65536 };
|
||||
|
||||
typedef struct {
|
||||
char parent[BOOTSTRAP_TEST_PATH_CAP];
|
||||
cbm_daemon_build_identity_t identity;
|
||||
pid_t children[BOOTSTRAP_ENOSPC_MAX_CHILDREN];
|
||||
size_t child_count;
|
||||
size_t spawn_calls;
|
||||
} bootstrap_enospc_host_t;
|
||||
|
||||
/* The production spawn exec's the product binary; this one forks a REAL daemon
|
||||
* host (cbm_daemon_host_run, the same entry `--cbm-daemon-internal` reaches)
|
||||
* whose record publication fails with ENOSPC through the inherited seam. */
|
||||
static bool bootstrap_enospc_host_spawn(void *opaque,
|
||||
const cbm_daemon_bootstrap_launch_spec_t *spec) {
|
||||
bootstrap_enospc_host_t *state = opaque;
|
||||
if (!spec || !spec->detached) {
|
||||
return false;
|
||||
}
|
||||
state->spawn_calls++;
|
||||
if (state->child_count >= BOOTSTRAP_ENOSPC_MAX_CHILDREN) {
|
||||
/* RED-run guard only: a client that keeps respawning a doomed daemon
|
||||
* for its whole deadline must not fork without bound. */
|
||||
return true;
|
||||
}
|
||||
pid_t child = fork();
|
||||
if (child < 0) {
|
||||
return false;
|
||||
}
|
||||
if (child == 0) {
|
||||
cbm_daemon_ipc_endpoint_t *endpoint = cbm_daemon_bootstrap_endpoint_new(state->parent);
|
||||
atomic_int stop_requested = ATOMIC_VAR_INIT(0);
|
||||
cbm_daemon_host_config_t config = {
|
||||
.endpoint = endpoint,
|
||||
.identity = state->identity,
|
||||
.executable_path = "/enospc-host-test",
|
||||
.stop_requested = &stop_requested,
|
||||
};
|
||||
int run_result = endpoint ? cbm_daemon_host_run(&config) : 0;
|
||||
_exit(run_result == -1 ? 0 : 50);
|
||||
}
|
||||
state->children[state->child_count++] = child;
|
||||
return true;
|
||||
}
|
||||
|
||||
static void bootstrap_enospc_reap(bootstrap_enospc_host_t *state, int *nonzero_exits) {
|
||||
*nonzero_exits = 0;
|
||||
for (size_t i = 0; i < state->child_count; i++) {
|
||||
int status = 0;
|
||||
pid_t waited;
|
||||
do {
|
||||
waited = waitpid(state->children[i], &status, WNOHANG);
|
||||
} while (waited < 0 && errno == EINTR);
|
||||
if (waited == 0) {
|
||||
(void)kill(state->children[i], SIGKILL);
|
||||
do {
|
||||
waited = waitpid(state->children[i], &status, 0);
|
||||
} while (waited < 0 && errno == EINTR);
|
||||
}
|
||||
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
|
||||
(*nonzero_exits)++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool bootstrap_read_file(const char *path, char *out, size_t capacity) {
|
||||
FILE *file = cbm_fopen(path, "rb");
|
||||
if (!file) {
|
||||
return false;
|
||||
}
|
||||
size_t used = fread(out, 1, capacity - 1, file);
|
||||
out[used] = '\0';
|
||||
(void)fclose(file);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* The record contract behind the fail-fast: a real listener failure in this
|
||||
* process is recorded and read back with stage, errno, and path; a record
|
||||
* older than the reader's spawn or for another endpoint is not evidence. */
|
||||
TEST(daemon_bootstrap_start_failure_record_round_trip) {
|
||||
bootstrap_endpoint_fixture_t fixture;
|
||||
bootstrap_endpoint_fixture_t other;
|
||||
ASSERT_TRUE(bootstrap_endpoint_fixture_start(&fixture, "failure-record"));
|
||||
char other_parent[BOOTSTRAP_TEST_PATH_CAP];
|
||||
int other_written =
|
||||
snprintf(other_parent, sizeof(other_parent), "%s/other-XXXXXX", fixture.parent);
|
||||
ASSERT(other_written > 0 && other_written < (int)sizeof(other_parent));
|
||||
ASSERT_TRUE(cbm_mkdtemp(other_parent) != NULL);
|
||||
memset(&other, 0, sizeof(other));
|
||||
other.endpoint = cbm_daemon_ipc_endpoint_new("1828000000000002", other_parent);
|
||||
ASSERT_TRUE(other.endpoint != NULL);
|
||||
char logs[BOOTSTRAP_TEST_PATH_CAP];
|
||||
int logs_written = snprintf(logs, sizeof(logs), "%s/logs", fixture.parent);
|
||||
ASSERT(logs_written > 0 && logs_written < (int)sizeof(logs));
|
||||
char expected_path[BOOTSTRAP_TEST_PATH_CAP];
|
||||
int path_written = snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp",
|
||||
cbm_daemon_ipc_endpoint_address(fixture.endpoint));
|
||||
ASSERT(path_written > 0 && path_written < (int)sizeof(expected_path));
|
||||
|
||||
cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC);
|
||||
cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(fixture.endpoint);
|
||||
cbm_daemon_ipc_posix_record_write_failure_set_for_test(0);
|
||||
ASSERT_TRUE(listener == NULL);
|
||||
|
||||
uint64_t now_s = (uint64_t)time(NULL);
|
||||
bool recorded = cbm_daemon_bootstrap_start_failure_record(logs, fixture.endpoint, "runtime");
|
||||
cbm_daemon_bootstrap_start_failure_t failure;
|
||||
int found =
|
||||
cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s - 2, &failure);
|
||||
cbm_daemon_bootstrap_start_failure_t stale;
|
||||
int stale_found =
|
||||
cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s + 60, &stale);
|
||||
cbm_daemon_bootstrap_start_failure_t foreign;
|
||||
int foreign_found =
|
||||
cbm_daemon_bootstrap_start_failure_read(logs, other.endpoint, now_s - 2, &foreign);
|
||||
char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE];
|
||||
cbm_daemon_bootstrap_start_failure_format(&failure, logs, message, sizeof(message));
|
||||
|
||||
cbm_daemon_ipc_endpoint_free(other.endpoint);
|
||||
(void)th_rmtree(fixture.parent);
|
||||
bootstrap_endpoint_fixture_finish(&fixture);
|
||||
|
||||
ASSERT_TRUE(recorded);
|
||||
ASSERT_EQ(found, 1);
|
||||
ASSERT_STR_EQ(failure.component, "runtime");
|
||||
ASSERT_STR_EQ(failure.stage, "pending_publication");
|
||||
ASSERT_EQ(failure.errno_value, ENOSPC);
|
||||
ASSERT_STR_EQ(failure.path, expected_path);
|
||||
ASSERT_TRUE(failure.pid == (uint64_t)getpid());
|
||||
ASSERT_EQ(stale_found, 0);
|
||||
ASSERT_EQ(foreign_found, 0);
|
||||
ASSERT_TRUE(strstr(message, "CBM daemon failed to start: pending_publication failed with "
|
||||
"ENOSPC (") != NULL);
|
||||
ASSERT_TRUE(strstr(message, expected_path) != NULL);
|
||||
ASSERT_TRUE(strstr(message, "/cbm-daemon.log") != NULL);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* #1828: a daemon that dies at publication (full /tmp) left every client
|
||||
* waiting the full 30 s and then reporting "active or starting" -- the
|
||||
* opposite of the truth. A real host is spawned against a runtime directory
|
||||
* whose record writes fail with ENOSPC; the client must report "failed to
|
||||
* start" naming the errno and the path by ending the wait on the recorded
|
||||
* cause, not by exhausting the deadline. The proof is the surfaced record and
|
||||
* the single spawn (see the assertions), never a wall-clock measurement. */
|
||||
TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication) {
|
||||
const char *old_cache = getenv("CBM_CACHE_DIR");
|
||||
char *saved_cache = old_cache ? cbm_strdup(old_cache) : NULL;
|
||||
bool snapshot_ok = !old_cache || saved_cache;
|
||||
|
||||
bootstrap_endpoint_fixture_t fixture;
|
||||
bool fixture_ok = snapshot_ok && bootstrap_endpoint_fixture_start(&fixture, "enospc-host");
|
||||
char cache[BOOTSTRAP_TEST_PATH_CAP] = {0};
|
||||
char daemon_log[BOOTSTRAP_TEST_PATH_CAP] = {0};
|
||||
char expected_path[BOOTSTRAP_TEST_PATH_CAP] = {0};
|
||||
int cache_written =
|
||||
fixture_ok ? snprintf(cache, sizeof(cache), "%s/cache", fixture.parent) : -1;
|
||||
int log_written =
|
||||
fixture_ok ? snprintf(daemon_log, sizeof(daemon_log), "%s/logs/cbm-daemon.log", cache) : -1;
|
||||
int path_written = fixture_ok ? snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp",
|
||||
cbm_daemon_ipc_endpoint_address(fixture.endpoint))
|
||||
: -1;
|
||||
bool environment_ready = cache_written > 0 && cache_written < (int)sizeof(cache) &&
|
||||
log_written > 0 && log_written < (int)sizeof(daemon_log) &&
|
||||
path_written > 0 && path_written < (int)sizeof(expected_path) &&
|
||||
cbm_mkdir_p(cache, 0700) && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0;
|
||||
|
||||
char self_build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE] = {0};
|
||||
bool identity_ready = environment_ready && cbm_daemon_runtime_process_build_fingerprint(
|
||||
(uint64_t)getpid(), self_build);
|
||||
static bootstrap_enospc_host_t host;
|
||||
memset(&host, 0, sizeof(host));
|
||||
(void)snprintf(host.parent, sizeof(host.parent), "%s", fixture_ok ? fixture.parent : "");
|
||||
host.identity = bootstrap_identity("2.4.0", self_build);
|
||||
|
||||
cbm_daemon_bootstrap_config_t config = {
|
||||
.role = CBM_DAEMON_PROCESS_MCP_CLIENT,
|
||||
.endpoint = fixture.endpoint,
|
||||
.identity = &host.identity,
|
||||
.executable_path = "/enospc-host-test",
|
||||
.connect_timeout_ms = 200,
|
||||
.startup_timeout_ms = 30000,
|
||||
};
|
||||
cbm_daemon_bootstrap_result_t result;
|
||||
memset(&result, 0, sizeof(result));
|
||||
cbm_daemon_bootstrap_status_t status = CBM_DAEMON_BOOTSTRAP_FAILED;
|
||||
if (identity_ready) {
|
||||
cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC);
|
||||
cbm_daemon_bootstrap_spawn_override_set_for_test(bootstrap_enospc_host_spawn, &host);
|
||||
status = cbm_daemon_bootstrap_execute(&config, &result);
|
||||
cbm_daemon_bootstrap_spawn_override_set_for_test(NULL, NULL);
|
||||
cbm_daemon_ipc_posix_record_write_failure_set_for_test(0);
|
||||
}
|
||||
int nonzero_exits = 0;
|
||||
bootstrap_enospc_reap(&host, &nonzero_exits);
|
||||
|
||||
static char log[BOOTSTRAP_ENOSPC_LOG_CAP];
|
||||
log[0] = '\0';
|
||||
bool log_read = bootstrap_read_file(daemon_log, log, sizeof(log));
|
||||
const char *listen_failed = strstr(log, "msg=daemon.ipc.listen_failed");
|
||||
bool daemon_named_cause = listen_failed && strstr(listen_failed, "errno=ENOSPC") != NULL &&
|
||||
strstr(listen_failed, expected_path) != NULL;
|
||||
bool message_names_failure = strstr(result.message, "failed to start") != NULL;
|
||||
bool message_names_errno = strstr(result.message, "ENOSPC") != NULL;
|
||||
bool message_names_path = strstr(result.message, expected_path) != NULL;
|
||||
bool stale_wording = strstr(result.message, "active or starting") != NULL;
|
||||
|
||||
if (saved_cache) {
|
||||
(void)cbm_setenv("CBM_CACHE_DIR", saved_cache, 1);
|
||||
} else {
|
||||
(void)cbm_unsetenv("CBM_CACHE_DIR");
|
||||
}
|
||||
free(saved_cache);
|
||||
if (fixture_ok) {
|
||||
(void)th_rmtree(fixture.parent);
|
||||
bootstrap_endpoint_fixture_finish(&fixture);
|
||||
}
|
||||
|
||||
ASSERT_TRUE(snapshot_ok);
|
||||
ASSERT_TRUE(fixture_ok);
|
||||
ASSERT_TRUE(environment_ready);
|
||||
ASSERT_TRUE(identity_ready);
|
||||
ASSERT_EQ(status, CBM_DAEMON_BOOTSTRAP_FAILED);
|
||||
ASSERT_TRUE(result.daemon_spawned);
|
||||
ASSERT_TRUE(host.child_count >= 1);
|
||||
ASSERT_TRUE(log_read);
|
||||
ASSERT_TRUE(daemon_named_cause);
|
||||
/* Fast-fail is proven by the MECHANISM, never by wall-clock (O9: a gate
|
||||
* never asserts a transient timing window). The recorded ENOSPC cause is
|
||||
* surfaced verbatim ("failed to start" + errno + path) and the slow
|
||||
* "active or starting" timeout wording is absent -- that message is emitted
|
||||
* ONLY on the fast-fail break (cbm_daemon_bootstrap_start_failure_format),
|
||||
* never on the 30 s deadline path -- and the client stopped after exactly
|
||||
* one spawn instead of respawning a doomed daemon until the deadline. Any
|
||||
* regression to the pre-#1828 30 s hang trips these deterministically. */
|
||||
ASSERT_FALSE(stale_wording);
|
||||
ASSERT_TRUE(message_names_failure);
|
||||
ASSERT_TRUE(message_names_errno);
|
||||
ASSERT_TRUE(message_names_path);
|
||||
ASSERT_EQ(host.child_count, 1U);
|
||||
ASSERT_EQ(nonzero_exits, 0);
|
||||
bootstrap_endpoint_fixture_finish(&fixture);
|
||||
PASS();
|
||||
}
|
||||
#endif
|
||||
|
||||
SUITE(daemon_bootstrap) {
|
||||
RUN_TEST(daemon_bootstrap_classifies_default_and_ui_as_mcp_clients);
|
||||
RUN_TEST(daemon_bootstrap_classifies_stateless_commands_without_client);
|
||||
@@ -903,4 +1162,8 @@ SUITE(daemon_bootstrap) {
|
||||
#ifdef __APPLE__
|
||||
RUN_TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous);
|
||||
#endif
|
||||
#ifndef _WIN32
|
||||
RUN_TEST(daemon_bootstrap_start_failure_record_round_trip);
|
||||
RUN_TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include "daemon/ipc_internal.h"
|
||||
#include "foundation/compat.h"
|
||||
#include "foundation/compat_thread.h"
|
||||
#include "foundation/log.h"
|
||||
#include "foundation/platform.h"
|
||||
#include "foundation/private_file_lock_internal.h"
|
||||
#include "foundation/subprocess.h"
|
||||
@@ -5187,6 +5188,83 @@ TEST(daemon_ipc_posix_single_uid_userns_real_smoke_issue1830) {
|
||||
#endif /* CBM_ENABLE_TEST_SEAMS */
|
||||
#endif /* !_WIN32 */
|
||||
|
||||
#ifndef _WIN32
|
||||
enum { IPC_TEST_LOG_CAPTURE_CAP = 16384 };
|
||||
static char ipc_test_log_capture[IPC_TEST_LOG_CAPTURE_CAP];
|
||||
static size_t ipc_test_log_capture_used;
|
||||
|
||||
static void ipc_test_log_capture_sink(const char *line) {
|
||||
if (!line) {
|
||||
return;
|
||||
}
|
||||
size_t length = strlen(line);
|
||||
if (ipc_test_log_capture_used + length + 2 > sizeof(ipc_test_log_capture)) {
|
||||
return;
|
||||
}
|
||||
memcpy(ipc_test_log_capture + ipc_test_log_capture_used, line, length);
|
||||
ipc_test_log_capture_used += length;
|
||||
ipc_test_log_capture[ipc_test_log_capture_used++] = '\n';
|
||||
ipc_test_log_capture[ipc_test_log_capture_used] = '\0';
|
||||
}
|
||||
|
||||
/* #1828: a full /tmp made every daemon start die at pending publication, and
|
||||
* the only durable trace was `daemon.ipc.listen_failed stage=pending_publication`
|
||||
* -- no syscall, no errno, no path. The reporter needed hours (and a wrong
|
||||
* `df` on the wrong mount) to find the cause. The failure line must name the
|
||||
* errno and the exact artifact path that could not be written. */
|
||||
TEST(daemon_ipc_listen_failure_names_errno_and_path) {
|
||||
static const char key[] = "1828000000000001";
|
||||
char parent[TEST_PATH_CAP] = {0};
|
||||
char runtime_dir[TEST_PATH_CAP] = {0};
|
||||
char socket_path[TEST_PATH_CAP] = {0};
|
||||
char pending_path[TEST_PATH_CAP] = {0};
|
||||
char expected_path[TEST_PATH_CAP + 16] = {0};
|
||||
cbm_daemon_ipc_endpoint_t *endpoint = NULL;
|
||||
cbm_daemon_ipc_listener_t *listener = NULL;
|
||||
|
||||
bool parent_ok = ipc_test_parent_new(parent, "enospc-diag");
|
||||
if (parent_ok) {
|
||||
endpoint = cbm_daemon_ipc_endpoint_new(key, parent);
|
||||
}
|
||||
if (endpoint) {
|
||||
ipc_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint));
|
||||
ipc_test_copy_path(socket_path, cbm_daemon_ipc_endpoint_address(endpoint));
|
||||
}
|
||||
bool paths_ok = endpoint && ipc_test_socket_pending_path(pending_path, socket_path) &&
|
||||
snprintf(expected_path, sizeof(expected_path), "path=%s.tmp", pending_path) > 0;
|
||||
if (paths_ok) {
|
||||
ipc_test_log_capture_used = 0;
|
||||
ipc_test_log_capture[0] = '\0';
|
||||
cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC);
|
||||
cbm_log_set_sink_ex(ipc_test_log_capture_sink, CBM_LOG_SINK_REPLACE);
|
||||
listener = cbm_daemon_ipc_listen(endpoint);
|
||||
cbm_log_set_sink(NULL);
|
||||
cbm_daemon_ipc_posix_record_write_failure_set_for_test(0);
|
||||
}
|
||||
const char *failed = strstr(ipc_test_log_capture, "msg=daemon.ipc.listen_failed");
|
||||
bool stage_named = failed && strstr(failed, "stage=pending_publication") != NULL;
|
||||
bool errno_named = failed && strstr(failed, "errno=ENOSPC") != NULL;
|
||||
bool path_named = failed && strstr(failed, expected_path) != NULL;
|
||||
struct stat leftover;
|
||||
bool namespace_clean = paths_ok && lstat(socket_path, &leftover) != 0 && errno == ENOENT &&
|
||||
lstat(pending_path, &leftover) != 0 && errno == ENOENT;
|
||||
|
||||
cbm_daemon_ipc_listener_close(listener);
|
||||
cbm_daemon_ipc_endpoint_free(endpoint);
|
||||
th_cleanup(parent_ok ? parent : NULL);
|
||||
|
||||
ASSERT_TRUE(parent_ok);
|
||||
ASSERT_TRUE(paths_ok);
|
||||
ASSERT_TRUE(listener == NULL);
|
||||
ASSERT_TRUE(failed != NULL);
|
||||
ASSERT_TRUE(stage_named);
|
||||
ASSERT_TRUE(errno_named);
|
||||
ASSERT_TRUE(path_named);
|
||||
ASSERT_TRUE(namespace_clean);
|
||||
PASS();
|
||||
}
|
||||
#endif
|
||||
|
||||
SUITE(daemon_ipc) {
|
||||
RUN_TEST(daemon_ipc_pending_timeout_race_returns_completed_io);
|
||||
RUN_TEST(daemon_ipc_pending_wait_failure_cancels_and_drains);
|
||||
@@ -5260,5 +5338,8 @@ SUITE(daemon_ipc) {
|
||||
RUN_TEST(daemon_ipc_posix_private_directory_rejects_world_writable_ancestor);
|
||||
RUN_TEST(daemon_ipc_posix_private_log_rejects_symlinks_and_is_owner_only);
|
||||
RUN_TEST(daemon_ipc_posix_rejects_non_socket_and_symlink_endpoints);
|
||||
#ifndef _WIN32
|
||||
RUN_TEST(daemon_ipc_listen_failure_names_errno_and_path);
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user