From 02eef7e82b3cde71dac1325967bb9d47c64ad4ee Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Tue, 15 Sep 2026 19:18:13 -0400 Subject: [PATCH] Make actor template resync interval configurable (#1522) Actor template discovery currently uses a fixed 20-second resync interval. This PR adds `--template-resync-interval` so deployments can tune that delay, preserving the `20s` default and rejecting nonpositive values. The setting also controls the reconciler's existing fallback retry delay. This is intentionally a small change to start a discussion about how template builds should be dispatched as the template catalog grows. The current resync fetches and decodes every template, including completed ones. At 100,000 templates, a 20-second interval implies roughly 5,000 template rows read per second per replica, assuming scans finish quickly. This is an estimate from the code, not a benchmark. Possible follow-ups: - **Immediate enqueue:** start work after creation, retaining a slower recovery scan for crashes between persistence and enqueue. - **Outbox/watch:** consume changes instead of scanning the catalog. The existing worker outbox still polls every 50 ms and broadcasts events to each subscriber; write overhead and recovery scans need consideration. - **Durable pending-build queue:** atomically record work, claim due jobs using short `FOR UPDATE SKIP LOCKED` transactions, and recover expired leases. This still polls, but queries pending work rather than the full catalog. `LISTEN/NOTIFY` is not the proposed default because it serializes notifying commits. - **Imperative build / long-running operation:** give callers explicit build control or a progress/completion handle. Either still needs reliable execution underneath. The main question is whether template builds need a broadcast change feed or a queue where replicas claim different jobs. Neither alternative is implemented here. [Full research: database costs, execution options, recovery requirements, and sources](https://gist.github.com/EItanya/0a1d6893ede34f2f0e9d9d1929ecb823). Validation: all Go race tests and repository verifiers passed using module mode with `NO_COLOR` unset. CLI checks confirmed the default and rejection of zero/negative intervals; the existing reconciliation test checks a custom interval. Fixes # > It's a good idea to open an issue first for discussion. - [ ] Tests pass - [ ] Appropriate changes to documentation are included in the PR --------- Signed-off-by: Eitan Yarmush --- .../controlapi/template_reconciler.go | 21 ++++++++++--------- .../controlapi/template_reconciler_test.go | 6 +++--- cmd/ateapi/main.go | 9 +++++++- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 132fa034f..831a1cd06 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -33,8 +33,7 @@ import ( ) const ( - templateResyncInterval = 20 * time.Second - templateListPageSize = 100 + templateListPageSize = 100 // templateWorkerCount is the number of goroutines draining the work // queue. @@ -74,15 +73,17 @@ type goldenActorControl interface { // ActorTemplateReconciler drives stored ActorTemplates through the golden // actor state machine. type ActorTemplateReconciler struct { - persistence templateReconcilerStore - control goldenActorControl - queue workqueue.TypedRateLimitingInterface[resources.ActorTemplateRef] + persistence templateReconcilerStore + control goldenActorControl + queue workqueue.TypedRateLimitingInterface[resources.ActorTemplateRef] + resyncInterval time.Duration } -func NewActorTemplateReconciler(persistence templateReconcilerStore, control goldenActorControl) *ActorTemplateReconciler { +func NewActorTemplateReconciler(persistence templateReconcilerStore, control goldenActorControl, resyncInterval time.Duration) *ActorTemplateReconciler { return &ActorTemplateReconciler{ - persistence: persistence, - control: control, + persistence: persistence, + control: control, + resyncInterval: resyncInterval, // Create rate-limiting queue with exponential backoff queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[resources.ActorTemplateRef]()), } @@ -96,7 +97,7 @@ func (r *ActorTemplateReconciler) Start(ctx context.Context) { for range templateWorkerCount { go wait.UntilWithContext(ctx, r.runWorker, time.Second) } - wait.UntilWithContext(ctx, r.resync, templateResyncInterval) + wait.UntilWithContext(ctx, r.resync, r.resyncInterval) }() } @@ -278,7 +279,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource return 0, r.fail(ctx, tmpl, reasonUnexpectedState, fmt.Sprintf("golden actor in unexpected state %v", state)) default: - return templateResyncInterval, nil + return r.resyncInterval, nil } } } diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index aac5806dc..7722b4bd3 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -299,7 +299,7 @@ func withFailed(reason string) func(*ateapipb.ActorTemplate) { } func newTestTemplateReconciler(persistence templateReconcilerStore, control goldenActorControl) *ActorTemplateReconciler { - return NewActorTemplateReconciler(persistence, control) + return NewActorTemplateReconciler(persistence, control, 7*time.Second) } func TestGoldenSnapshotWarmupFor(t *testing.T) { @@ -467,8 +467,8 @@ func TestReconcileOne(t *testing.T) { name: "unspecified actor state checks back later", template: testTemplate(), control: &fakeGoldenControl{exists: true, goldenState: ateapipb.ActorState_ACTOR_STATE_UNSPECIFIED}, - wantRequeueMin: templateResyncInterval, - wantRequeueMax: templateResyncInterval, + wantRequeueMin: 7 * time.Second, + wantRequeueMax: 7 * time.Second, }, { name: "lease conflict yields without error", diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 47c7ccebf..340ea5109 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -64,6 +64,8 @@ import ( // maxRPCDeadline is the max deadline for all RPC methods exposed by this server. const maxRPCDeadline = 10 * time.Minute +const minResyncInterval = 250 * time.Millisecond + var ( listenAddr = pflag.String("grpc-listen-addr", ":443", "Address and port the gRPC server should listen on.") metricsListenAddr = pflag.String("metrics-listen-addr", ":9090", "Address and port the prometheus metrics server should listen on.") @@ -83,6 +85,8 @@ var ( drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") + templateResyncInterval = pflag.Duration("template-resync-interval", 20*time.Second, fmt.Sprintf("Interval between actor template resyncs. Must be at least %s.", minResyncInterval)) + showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") ) @@ -99,6 +103,9 @@ func main() { serverboot.Fatal(ctx, "Invalid --log-level", err) } slog.InfoContext(ctx, "ateapi starting", slog.String("version", version.Version)) + if *templateResyncInterval < minResyncInterval { + serverboot.Fatal(ctx, "Invalid --template-resync-interval", fmt.Errorf("must be at least %s", minResyncInterval)) + } // Kept separate from ctx so that in-progress work (clients, informers) is // not cancelled the moment SIGTERM arrives. The drainOnShutdown @@ -226,7 +233,7 @@ func main() { ) // Drive stored ActorTemplates through the golden actor flow. - templateReconciler := controlapi.NewActorTemplateReconciler(persistence, controlSrv) + templateReconciler := controlapi.NewActorTemplateReconciler(persistence, controlSrv, *templateResyncInterval) templateReconciler.Start(shutdownCtx) lisCfg := &net.ListenConfig{}