fix: address QA round 1

F1 (major) — the failure state machine the backoff feeds had no test
coverage. Deleting the streak reset, or widening the sustained-failure
comparison, left all 7788 tests green: the index_backoff_* tests cover the
delay arithmetic, and nothing covered the counter that selects which delay
applies. A streak that never increments silently degrades the backoff to the
unbounded retry it exists to prevent.

Adds cbm_watcher_index_failure_count(), alongside the existing
cbm_watcher_watch_count() in watcher.h's "Introspection (for testing)"
section, and a test that drives a real git repo through fail/fail/succeed on
the failing_index_callback seam already used by the #937 test, asserting the
streak goes 0 -> 1 -> 2 -> 0. Verified by mutation: removing the reset turns
it red at the reset assertion while every index_backoff_* test stays green.

The delay-GATING half stays uncovered on purpose. Every integration test
calls cbm_watcher_touch first, which zeroes next_poll_ns, so whether the
scheduler honours the computed deadline is unobservable without an
injectable clock the watcher does not have. That harness is a larger change
than this fix.

F3 (minor) — cbm_watcher_index_backoff_ms was non-monotonic for an interval
above the ceiling: zero failures returned the interval, one failure returned
the smaller ceiling. Unreachable today (POLL_MAX_MS < the ceiling), but the
function is exported, so the clamp is now a floor as well as a cap and
backing off can never schedule sooner than the project's own cadence.

F2 (minor) needed no code change: a busy-skip following an unresolved
failure streak keeps the backed-off delay, which is deliberate — only a
success clears the streak, and treating rc > 0 as recovery would let a
project alternating fail/busy never back off at all. The PR description's
over-strong "busy-skip is behaviourally unchanged" claim is corrected there.

Refs #2015

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FbrM52pmB1m2SR1vRRoNnf
Signed-off-by: Shane McCarron <shane.mccarron@corvexconnect.com>

src/watcher/watcher.c

@@ -171,7 +171,15 @@ int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures) {
     int shift =
         consecutive_failures < INDEX_FAIL_SHIFT_MAX ? consecutive_failures : INDEX_FAIL_SHIFT_MAX;
     int64_t delay = (int64_t)interval_ms << shift;
-    return delay > INDEX_FAIL_CEILING_MS ? INDEX_FAIL_CEILING_MS : (int)delay;
+    if (delay > INDEX_FAIL_CEILING_MS) {
+        delay = INDEX_FAIL_CEILING_MS;
+    }
+    /* Backing off must never schedule SOONER than the project's own cadence.
+     * Unreachable today (POLL_MAX_MS < the ceiling), but clamping here keeps
+     * the function monotonic for every input rather than only for the inputs
+     * the current constants can produce — the caller's contract is "a delay
+     * that never shrinks as failures accumulate". */
+    return (int)(delay < interval_ms ? interval_ms : delay);
 }
 
 int cbm_watcher_poll_interval_ms(int file_count) {
@@ -1161,6 +1169,17 @@ void cbm_watcher_touch(cbm_watcher_t *w, const char *project_name) {
     cbm_mutex_unlock(&w->projects_lock);
 }
 
+int cbm_watcher_index_failure_count(cbm_watcher_t *w, const char *project_name) {
+    if (!w || !project_name) {
+        return -1;
+    }
+    cbm_mutex_lock(&w->projects_lock);
+    project_state_t *s = cbm_ht_get(w->projects, project_name);
+    int failures = s ? s->index_failures : -1;
+    cbm_mutex_unlock(&w->projects_lock);
+    return failures;
+}
+
 int cbm_watcher_watch_count(cbm_watcher_t *w) {
     if (!w) {
         return 0;

src/watcher/watcher.h

@@ -91,6 +91,13 @@ void cbm_watcher_stop(cbm_watcher_t *w);
 /* Return the number of projects in the watch list. */
 int cbm_watcher_watch_count(cbm_watcher_t *w);
 
+/* Return a watched project's consecutive hard-index-failure count, or -1 when
+ * it is not watched. Exposed because the failure state machine (increment on
+ * a hard error, reset on success) is otherwise unobservable from a test: the
+ * only other thing it feeds is a poll deadline, and gating on that deadline
+ * needs a controllable clock the watcher does not have. */
+int cbm_watcher_index_failure_count(cbm_watcher_t *w, const char *project_name);
+
 /* Return the adaptive poll interval (ms) for a given file count. */
 int cbm_watcher_poll_interval_ms(int file_count);
 

tests/test_watcher.c

@@ -137,6 +137,20 @@ TEST(index_backoff_degenerate_inputs) {
     PASS();
 }
 
+TEST(index_backoff_never_schedules_sooner_than_the_interval) {
+    /* An interval above the ceiling must not be SHORTENED by backing off:
+     * the clamp is a floor as well as a cap. Not reachable while
+     * POLL_MAX_MS < the ceiling, but the helper is exported, so its contract
+     * has to hold for what a caller can pass, not only for what today's
+     * constants produce. */
+    int over = 400000; /* > INDEX_FAIL_CEILING_MS */
+    ASSERT_EQ(cbm_watcher_index_backoff_ms(over, 0), over);
+    for (int failures = 1; failures < 20; failures++) {
+        ASSERT_TRUE(cbm_watcher_index_backoff_ms(over, failures) >= over);
+    }
+    PASS();
+}
+
 /* ══════════════════════════════════════════════════════════════════
  *  LIFECYCLE
  * ══════════════════════════════════════════════════════════════════ */
@@ -1788,6 +1802,92 @@ TEST(watcher_failed_reindex_retries_issue937) {
     PASS();
 }
 
+/* The failure state machine the backoff feeds (#2015): the streak must count
+ * consecutive HARD failures and reset the moment one reindex succeeds. The
+ * delay it computes is covered by the index_backoff_* unit tests; what is
+ * covered here is the counter that selects which delay applies, because a
+ * streak that never increments — or never resets — silently reverts the
+ * backoff to the unbounded retry it exists to prevent, with every other test
+ * still green. Each poll is preceded by cbm_watcher_touch, which zeroes
+ * next_poll_ns, so this exercises the counter without depending on a clock. */
+TEST(watcher_index_failure_streak_increments_and_resets_issue2015) {
+    char tmpdir[256];
+    snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_strk_XXXXXX");
+    if (!cbm_mkdtemp(tmpdir))
+        FAIL("cbm_mkdtemp failed");
+
+    if (wt_git(tmpdir, "init -q") != 0) {
+        th_rmtree(tmpdir);
+        FAIL("git init failed");
+    }
+    {
+        char p[300];
+        th_write_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "hello\n");
+    }
+    wt_git(tmpdir, "add file.txt");
+    wt_git(tmpdir, "commit -q -m init");
+
+    cbm_store_t *store = cbm_store_open_memory();
+    cbm_watcher_t *w = cbm_watcher_new(store, failing_index_callback, NULL);
+
+    cbm_watcher_watch(w, "strk-repo", tmpdir);
+    failing_index_calls = 0;
+    failing_index_fail_first_n = 2; /* first TWO reindex attempts fail hard */
+
+    /* Baseline (clean tree): no reindex, so no failures yet. */
+    cbm_watcher_poll_once(w);
+    ASSERT_EQ(failing_index_calls, 0);
+    ASSERT_EQ(cbm_watcher_index_failure_count(w, "strk-repo"), 0);
+
+    /* HEAD moves. */
+    {
+        char p[300];
+        th_append_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "world\n");
+    }
+    wt_git(tmpdir, "add file.txt");
+    wt_git(tmpdir, "commit -q -m add-world");
+
+    /* First hard failure → streak 1. */
+    cbm_watcher_touch(w, "strk-repo");
+    cbm_watcher_poll_once(w);
+    ASSERT_EQ(failing_index_calls, 1);
+    ASSERT_EQ(cbm_watcher_index_failure_count(w, "strk-repo"), 1);
+
+    /* Second consecutive hard failure → streak 2 (it accumulates; it is not
+     * a boolean, which is what makes the delay grow). */
+    cbm_watcher_touch(w, "strk-repo");
+    cbm_watcher_poll_once(w);
+    ASSERT_EQ(failing_index_calls, 2);
+    ASSERT_EQ(cbm_watcher_index_failure_count(w, "strk-repo"), 2);
+
+    /* Third attempt succeeds → streak resets, so a project that recovers
+     * returns to its normal cadence instead of staying backed off. */
+    cbm_watcher_touch(w, "strk-repo");
+    cbm_watcher_poll_once(w);
+    ASSERT_EQ(failing_index_calls, 3);
+    ASSERT_EQ(cbm_watcher_index_failure_count(w, "strk-repo"), 0);
+
+    cbm_watcher_free(w);
+    cbm_store_close(store);
+    th_rmtree(tmpdir);
+    PASS();
+}
+
+TEST(watcher_index_failure_count_unknown_project) {
+    cbm_store_t *store = cbm_store_open_memory();
+    cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL);
+
+    /* Unwatched and NULL inputs report -1, distinct from a watched project
+     * sitting at a legitimate 0. */
+    ASSERT_EQ(cbm_watcher_index_failure_count(w, "never-watched"), -1);
+    ASSERT_EQ(cbm_watcher_index_failure_count(w, NULL), -1);
+    ASSERT_EQ(cbm_watcher_index_failure_count(NULL, "x"), -1);
+
+    cbm_watcher_free(w);
+    cbm_store_close(store);
+    PASS();
+}
+
 TEST(watcher_multiple_projects) {
     /* Create two temporary git repos */
     char tmpdirA[256];
@@ -3200,6 +3300,9 @@ SUITE(watcher) {
     RUN_TEST(index_backoff_reaches_ceiling_and_stays);
     RUN_TEST(index_backoff_is_monotonic_and_bounded);
     RUN_TEST(index_backoff_degenerate_inputs);
+    RUN_TEST(index_backoff_never_schedules_sooner_than_the_interval);
+    RUN_TEST(watcher_index_failure_streak_increments_and_resets_issue2015);
+    RUN_TEST(watcher_index_failure_count_unknown_project);
 
     /* Lifecycle */
     RUN_TEST(watcher_create_free);