trending/substrate
1
0
Fork 0
mirror of https://github.com/agent-substrate/substrate.git synced 2026-10-03 23:12:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

e2e/parking: retry the parked request while the worker is still freeing

Browse Source 
SuspendActor returns before the suspend completes, and on the micro-VM
class the snapshot upload routinely outlives the 5s park budget under
CI contention — the router's budget-exhausted 503 is then correct
behavior, not a failure. Retry the request (bounded at 3 attempts):
each attempt parks anew and the completed suspend lets one resume the
actor. A stranded worker (#675's root cause) fails all attempts, so
the regression stays pinned.

This was the dominant historical failure mode of the #675 flake
(identical parking_test.go:113 503s in pre-PR runs 32305728993,
32397291519, 32487958077) — distinct from the mid-restore
cancellation that #991 fixes.
This commit is contained in:
Aditya Shantanu
2026-08-24 09:51:30 -07:00
parent 8a84e26dfa
commit d637690dc3
1 changed files with 39 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/e2e/suites/parking/parking_test.go
+39 -21
View File
@@ -80,34 +80,52 @@ func TestRequestParking(t *testing.T) {
resumeActor(ctx, t, clients, actorA)
waitForActorState(ctx, t, clients, actorA, ateapipb.ActorState_ACTOR_STATE_RUNNING)
// Request actor B: the pool is full, so the request parks.
// Request actor B: the pool is full, so the request parks. Freeing
// the worker is asynchronous — SuspendActor(A) returns before the
// suspend completes, and on the micro-VM class the snapshot upload
// routinely outlives the 5s park budget under CI contention. A
// budget-exhausted 503 while the suspend is still in flight is the
// router behaving correctly, so the request is retried: each attempt
// parks anew, and the suspend's completion lets one of them resume B.
// A stranded worker (#675's root cause) fails every attempt, so the
// regression this subtest pins still fails it.
type result struct {
resp *http.Response
body string
err error
}
resCh := make(chan result, 1)
start := time.Now()
go func() {
resp, err := router.Get(ctx, resources.ActorRef{Atespace: parkingAtespace, Name: actorB}, "/")
var body string
if err == nil {
b, _ := io.ReadAll(resp.Body)
resp.Body.Close()
body = string(b)
var res result
var elapsed time.Duration
for attempt := 1; ; attempt++ {
start := time.Now()
go func() {
resp, err := router.Get(ctx, resources.ActorRef{Atespace: parkingAtespace, Name: actorB}, "/")
var body string
if err == nil {
b, _ := io.ReadAll(resp.Body)
resp.Body.Close()
body = string(b)
}
resCh <- result{resp, body, err}
}()
if attempt == 1 {
// Free the worker only once the request is observably parked —
// the statusz gauge, not a sleep, is the synchronization point.
waitForParkedCount(ctx, t, statusz, func(active int) bool { return active >= 1 })
suspendActor(ctx, t, clients, actorA)
}
resCh <- result{resp, body, err}
}()
// Free the worker only once the request is observably parked — the
// statusz gauge, not a sleep, is the synchronization point.
waitForParkedCount(ctx, t, statusz, func(active int) bool { return active >= 1 })
suspendActor(ctx, t, clients, actorA)
res := <-resCh
elapsed := time.Since(start)
if res.err != nil {
t.Fatalf("parked request failed transport-level: %v", res.err)
res = <-resCh
elapsed = time.Since(start)
if res.err != nil {
t.Fatalf("parked request failed transport-level: %v", res.err)
}
if res.resp.StatusCode == http.StatusServiceUnavailable &&
strings.Contains(res.body, "no free workers available") && attempt < 3 {
t.Logf("attempt %d budget-exhausted while the worker was still freeing (503 after %v); retrying", attempt, elapsed)
continue
}
break
}
if res.resp.StatusCode != http.StatusOK {
t.Fatalf("parked request: status = %d (body %q), want 200", res.resp.StatusCode, res.body)