atenet/router: report budget exhaustion as its own parking outcome

Review feedback: folding park-budget exhaustion into the generic 'error' outcome hid the one signal operators need from this metric -- that requests waited the full budget and the pool never freed (capacity problem), as opposed to resumes failing outright (fault). The resumer now marks the surfaced capacity error with a wrapper that unwraps to the underlying gRPC status, so the HTTP mapping is unchanged and the wait-duration histogram gains a budget_exhausted outcome label.
This commit is contained in:
Omer Yahud
2026-07-28 23:55:51 -07:00
committed by Bowei Du
parent 4fc2ed20ed
commit 79365efeac
5 changed files with 34 additions and 10 deletions
+9 -6
View File
@@ -34,10 +34,11 @@ type parkOutcome string
// Park-wait outcomes, recorded on the parking.wait.duration histogram.
const (
parkOutcomeServed parkOutcome = "served" // resume succeeded and the request was routed
parkOutcomeTimeout parkOutcome = "timeout" // the request's deadline elapsed while parked
parkOutcomeCanceled parkOutcome = "canceled" // the client disconnected while parked
parkOutcomeError parkOutcome = "error" // resume failed (including park-budget exhaustion)
parkOutcomeServed parkOutcome = "served" // resume succeeded and the request was routed
parkOutcomeBudgetExhausted parkOutcome = "budget_exhausted" // the park budget elapsed while still blocked on a retryable condition
parkOutcomeTimeout parkOutcome = "timeout" // the request's deadline elapsed while parked
parkOutcomeCanceled parkOutcome = "canceled" // the client disconnected while parked
parkOutcomeError parkOutcome = "error" // resume failed
)
// parkingConfig controls how the router parks resume-gated requests.
@@ -141,12 +142,14 @@ func (l *parkingLot) status() ParkingStatus {
}
// parkOutcomeFor classifies a completed resume attempt for the wait-duration
// metric. A budget-exhausted park surfaces the underlying capacity error and is
// reported as parkOutcomeError.
// metric.
func parkOutcomeFor(err error) parkOutcome {
var budget *budgetExhaustedError
switch {
case err == nil:
return parkOutcomeServed
case errors.As(err, &budget):
return parkOutcomeBudgetExhausted
case errors.Is(err, context.Canceled):
return parkOutcomeCanceled
case errors.Is(err, context.DeadlineExceeded):
@@ -138,6 +138,7 @@ func TestParkOutcomeFor(t *testing.T) {
want parkOutcome
}{
{"nil is served", nil, parkOutcomeServed},
{"budget exhaustion is explicit", &budgetExhaustedError{cause: errOther}, parkOutcomeBudgetExhausted},
{"canceled", context.Canceled, parkOutcomeCanceled},
{"deadline is timeout", context.DeadlineExceeded, parkOutcomeTimeout},
{"other is error", errOther, parkOutcomeError},
+13 -2
View File
@@ -60,6 +60,16 @@ func resumeBackoff() wait.Backoff {
}
}
// budgetExhaustedError marks a resume that was still blocked on a retryable
// condition (e.g. "no free workers available") when the parking budget elapsed.
// It wraps the last retryable error, so the HTTP boundary still maps the
// underlying gRPC status faithfully (503 with the capacity message), while the
// parking metrics can report budget exhaustion as its own outcome.
type budgetExhaustedError struct{ cause error }
func (e *budgetExhaustedError) Error() string { return e.cause.Error() }
func (e *budgetExhaustedError) Unwrap() error { return e.cause }
// ActorResumer coordinates safe, deduplicated resumption of actors.
type ActorResumer struct {
apiClient ateapipb.ControlClient
@@ -159,9 +169,10 @@ func (r *ActorResumer) ResumeActor(ctx context.Context, actorRef resources.Actor
// If the budget elapsed (DeadlineExceeded) while we were still retrying a
// transient error, surface that underlying error rather than the generic
// wait error so the HTTP boundary maps it faithfully (e.g. 503 "no free
// workers available") instead of a misleading timeout.
// workers available") instead of a misleading timeout. The wrapper marks
// the exhaustion explicitly for the parking wait-duration metric.
if lastRetryErr != nil && (errors.Is(err, context.DeadlineExceeded) || wait.Interrupted(err)) {
return nil, lastRetryErr
return nil, &budgetExhaustedError{cause: lastRetryErr}
}
return nil, err
}
+7 -1
View File
@@ -16,6 +16,7 @@ package router
import (
"context"
"errors"
"sync"
"testing"
"time"
@@ -227,10 +228,15 @@ func TestActorResumer_Parking(t *testing.T) {
// the pool never frees up.
resumer := NewActorResumer(mock, withParking(true, 1500*time.Millisecond))
_, err := resumer.ResumeActor(context.Background(), testAtespace, testActorName)
// The client must see the meaningful capacity error, not a generic timeout.
// The client must see the meaningful capacity error, not a generic
// timeout: status.Code must unwrap through the budget-exhaustion marker.
if got := status.Code(err); got != codes.FailedPrecondition {
t.Errorf("expected FailedPrecondition after park budget elapsed, got %v (err=%v)", got, err)
}
var budget *budgetExhaustedError
if !errors.As(err, &budget) {
t.Errorf("expected the error to be marked as budget exhaustion, got %T (%v)", err, err)
}
mu.Lock()
defer mu.Unlock()
if calls < 2 {
+4 -1
View File
@@ -86,7 +86,10 @@ within the historical `15s` budget.
- `atenet.router.parking.active` — up/down counter: requests currently parked.
- `atenet.router.parking.wait.duration` — histogram (seconds) of time spent
parked, labeled `outcome` ∈ {`served`, `timeout`, `canceled`, `error`}.
parked, labeled `outcome` ∈ {`served`, `budget_exhausted`, `timeout`,
`canceled`, `error`}. `budget_exhausted` means the full park budget elapsed
while the pool stayed saturated — the signal that capacity, not a fault, is
the bottleneck.
- `atenet.router.parking.rejected` — counter: requests shed because the lot was
full.