mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-10-02 03:24:42 +08:00
atenet/router: report budget exhaustion as its own parking outcome
Review feedback: folding park-budget exhaustion into the generic 'error' outcome hid the one signal operators need from this metric -- that requests waited the full budget and the pool never freed (capacity problem), as opposed to resumes failing outright (fault). The resumer now marks the surfaced capacity error with a wrapper that unwraps to the underlying gRPC status, so the HTTP mapping is unchanged and the wait-duration histogram gains a budget_exhausted outcome label.
This commit is contained in:
@@ -34,10 +34,11 @@ type parkOutcome string
|
||||
|
||||
// Park-wait outcomes, recorded on the parking.wait.duration histogram.
|
||||
const (
|
||||
parkOutcomeServed parkOutcome = "served" // resume succeeded and the request was routed
|
||||
parkOutcomeTimeout parkOutcome = "timeout" // the request's deadline elapsed while parked
|
||||
parkOutcomeCanceled parkOutcome = "canceled" // the client disconnected while parked
|
||||
parkOutcomeError parkOutcome = "error" // resume failed (including park-budget exhaustion)
|
||||
parkOutcomeServed parkOutcome = "served" // resume succeeded and the request was routed
|
||||
parkOutcomeBudgetExhausted parkOutcome = "budget_exhausted" // the park budget elapsed while still blocked on a retryable condition
|
||||
parkOutcomeTimeout parkOutcome = "timeout" // the request's deadline elapsed while parked
|
||||
parkOutcomeCanceled parkOutcome = "canceled" // the client disconnected while parked
|
||||
parkOutcomeError parkOutcome = "error" // resume failed
|
||||
)
|
||||
|
||||
// parkingConfig controls how the router parks resume-gated requests.
|
||||
@@ -141,12 +142,14 @@ func (l *parkingLot) status() ParkingStatus {
|
||||
}
|
||||
|
||||
// parkOutcomeFor classifies a completed resume attempt for the wait-duration
|
||||
// metric. A budget-exhausted park surfaces the underlying capacity error and is
|
||||
// reported as parkOutcomeError.
|
||||
// metric.
|
||||
func parkOutcomeFor(err error) parkOutcome {
|
||||
var budget *budgetExhaustedError
|
||||
switch {
|
||||
case err == nil:
|
||||
return parkOutcomeServed
|
||||
case errors.As(err, &budget):
|
||||
return parkOutcomeBudgetExhausted
|
||||
case errors.Is(err, context.Canceled):
|
||||
return parkOutcomeCanceled
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
|
||||
@@ -138,6 +138,7 @@ func TestParkOutcomeFor(t *testing.T) {
|
||||
want parkOutcome
|
||||
}{
|
||||
{"nil is served", nil, parkOutcomeServed},
|
||||
{"budget exhaustion is explicit", &budgetExhaustedError{cause: errOther}, parkOutcomeBudgetExhausted},
|
||||
{"canceled", context.Canceled, parkOutcomeCanceled},
|
||||
{"deadline is timeout", context.DeadlineExceeded, parkOutcomeTimeout},
|
||||
{"other is error", errOther, parkOutcomeError},
|
||||
|
||||
@@ -60,6 +60,16 @@ func resumeBackoff() wait.Backoff {
|
||||
}
|
||||
}
|
||||
|
||||
// budgetExhaustedError marks a resume that was still blocked on a retryable
|
||||
// condition (e.g. "no free workers available") when the parking budget elapsed.
|
||||
// It wraps the last retryable error, so the HTTP boundary still maps the
|
||||
// underlying gRPC status faithfully (503 with the capacity message), while the
|
||||
// parking metrics can report budget exhaustion as its own outcome.
|
||||
type budgetExhaustedError struct{ cause error }
|
||||
|
||||
func (e *budgetExhaustedError) Error() string { return e.cause.Error() }
|
||||
func (e *budgetExhaustedError) Unwrap() error { return e.cause }
|
||||
|
||||
// ActorResumer coordinates safe, deduplicated resumption of actors.
|
||||
type ActorResumer struct {
|
||||
apiClient ateapipb.ControlClient
|
||||
@@ -159,9 +169,10 @@ func (r *ActorResumer) ResumeActor(ctx context.Context, actorRef resources.Actor
|
||||
// If the budget elapsed (DeadlineExceeded) while we were still retrying a
|
||||
// transient error, surface that underlying error rather than the generic
|
||||
// wait error so the HTTP boundary maps it faithfully (e.g. 503 "no free
|
||||
// workers available") instead of a misleading timeout.
|
||||
// workers available") instead of a misleading timeout. The wrapper marks
|
||||
// the exhaustion explicitly for the parking wait-duration metric.
|
||||
if lastRetryErr != nil && (errors.Is(err, context.DeadlineExceeded) || wait.Interrupted(err)) {
|
||||
return nil, lastRetryErr
|
||||
return nil, &budgetExhaustedError{cause: lastRetryErr}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -227,10 +228,15 @@ func TestActorResumer_Parking(t *testing.T) {
|
||||
// the pool never frees up.
|
||||
resumer := NewActorResumer(mock, withParking(true, 1500*time.Millisecond))
|
||||
_, err := resumer.ResumeActor(context.Background(), testAtespace, testActorName)
|
||||
// The client must see the meaningful capacity error, not a generic timeout.
|
||||
// The client must see the meaningful capacity error, not a generic
|
||||
// timeout: status.Code must unwrap through the budget-exhaustion marker.
|
||||
if got := status.Code(err); got != codes.FailedPrecondition {
|
||||
t.Errorf("expected FailedPrecondition after park budget elapsed, got %v (err=%v)", got, err)
|
||||
}
|
||||
var budget *budgetExhaustedError
|
||||
if !errors.As(err, &budget) {
|
||||
t.Errorf("expected the error to be marked as budget exhaustion, got %T (%v)", err, err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if calls < 2 {
|
||||
|
||||
@@ -86,7 +86,10 @@ within the historical `15s` budget.
|
||||
|
||||
- `atenet.router.parking.active` — up/down counter: requests currently parked.
|
||||
- `atenet.router.parking.wait.duration` — histogram (seconds) of time spent
|
||||
parked, labeled `outcome` ∈ {`served`, `timeout`, `canceled`, `error`}.
|
||||
parked, labeled `outcome` ∈ {`served`, `budget_exhausted`, `timeout`,
|
||||
`canceled`, `error`}. `budget_exhausted` means the full park budget elapsed
|
||||
while the pool stayed saturated — the signal that capacity, not a fault, is
|
||||
the bottleneck.
|
||||
- `atenet.router.parking.rejected` — counter: requests shed because the lot was
|
||||
full.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user