Files
Sneha-at 6a552d590d Add a default cluster snapshot storage location with snapshotConfig.storageLocation override in preview mode (#2544)
> The PR title becomes the release note. Write it for users: what
changed for them, not how the code changed.

#### Breaking change

ate-api-server now stores snapshots under a new, operator-set
`--snapshot-location`
(for example `gs://BUCKET/` or `gs://BUCKET/PREFIX/`).
`ActorTemplate.snapshotConfig.storageLocation`
is now a preview field, honored only with
`--preview=PerTemplateSnapshotLocation`.
`snapshotConfig` itself is now optional; an omitted one takes every
default.

What you must do when upgrading:

- **ate-setup installs:** set `SNAPSHOT_LOCATION` (`--snapshot-location`
/ `storage.snapshotLocation`).
ate-setup passes it to ate-api-server and, unless the
`PerTemplateSnapshotLocation` preview gate
is enabled, fails before touching the cluster if it is unset.
`BUCKET_NAME` alone is not enough.
The kind profile defaults it to `gs://$BUCKET_NAME/`
(`gs://ate-snapshots/`). ate-setup does not
create the bucket or grant access to it on GKE (`go run
./tools/setup-gcp bootstrap` does).
- **Custom deployments of ate-api-server:** add `--snapshot-location`
(or `ATE_API_SNAPSHOT_LOCATION`
with `--snapshot-location=@env`). Without it, the server refuses to
start unless the
  `PerTemplateSnapshotLocation` preview gate is enabled.
- **Templates that set `storageLocation`:** without the gate, that value
is ignored and new
snapshots and tags go under `--snapshot-location`. If you rely on
per-template buckets
or paths (IAM, retention, billing), enable
`--preview=PerTemplateSnapshotLocation` to keep the
old behavior. With the gate, the field stays optional: a template that
leaves it empty uses
`--snapshot-location`, and `CreateActorTemplate` fails with
`FailedPrecondition` if neither is set.
- **Clients that set `storageLocation` with the gate enabled:** they
must opt in to preview
features (`kubectl ate --preview`, or the `Ate-Preview-Allowed: true`
header). Otherwise
  `CreateActorTemplate` fails with `InvalidArgument`.
- **Existing actors:** snapshots already taken keep working for resume.
If an actor's snapshot
location changes (by upgrading with the gate off, changing the flag, or
toggling the gate),
its next suspend writes under the new location and releases the previous
snapshot from the
old one. Until that suspend, changing the actor's `actor_template` is
rejected, and deleting the
actor collects its snapshots under the old location. If releasing the
previous snapshot
fails, its objects stay under the old location and a later delete does
not collect them;
ate-api-server logs a warning naming the snapshot URI. To keep the old
locations, enable the
  gate before upgrading.

ActorTemplate is not the right place to specify snapshot location.
Bucket is an implementation detail of snapshots, workloads shouldn’t
need to care about it. It is the responsibility of substrate admins to
provision the buckets and give access to Substrate. This PR adds a
cluster wide storage location as a building step towards the final goal
of adding these details in a separate SnapshotStorageConfig.


- [x] Tests pass
- [x] Appropriate changes to documentation are included in the PR
2026-10-19 19:36:35 +00:00

168 lines
7.0 KiB
Go

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controlapi
import (
"context"
"errors"
"fmt"
"go.ate.dev/substrate/cmd/ateapi/internal/apivalidation"
"go.ate.dev/substrate/cmd/ateapi/internal/defaults"
"go.ate.dev/substrate/cmd/ateapi/internal/store"
"go.ate.dev/substrate/internal/apierror"
"go.ate.dev/substrate/internal/resources"
"go.ate.dev/substrate/pkg/proto/ateapipb"
"google.golang.org/protobuf/proto"
"k8s.io/apimachinery/pkg/util/validation/field"
)
func (s *RPCService) CreateActorTemplate(ctx context.Context, req *ateapipb.CreateActorTemplateRequest) (*ateapipb.ActorTemplate, error) {
// First scrub any fields that users are not allowed to set, then fill the
// defaults so validation sees the final resource state.
in := req.GetActorTemplate()
if in != nil { // otherwise validation will flag it
scrubResourceMetadataForCreate(in.Metadata)
in.Status = nil
defaults.Apply(in)
}
// Validate the request, including the object within it.
if errs := apivalidation.ValidateCreateActorTemplateRequest(ctx, req); len(errs) > 0 {
return nil, resources.ToAPIError(errs)
}
templateRef := resources.ActorTemplateRefFromActorTemplate(in)
stored, err := s.impl.CreateActorTemplate(ctx, in)
if err != nil {
if errors.Is(err, store.ErrAlreadyExists) {
return nil, apierror.AlreadyExists("ActorTemplate %s already exists", templateRef)
}
if errors.Is(err, store.ErrFailedPrecondition) {
return nil, apierror.FailedPrecondition("%v", err)
}
return nil, fmt.Errorf("while recording actor template: %w", err)
}
return stored, nil
}
func (s *ServiceImpl) CreateActorTemplate(ctx context.Context, inTemplate *ateapipb.ActorTemplate) (*ateapipb.ActorTemplate, error) {
// Build the stored object: status is server-owned and starts empty.
outTemplate := proto.Clone(inTemplate).(*ateapipb.ActorTemplate)
outTemplate.Status = &ateapipb.ActorTemplateStatus{}
// Validate the final value before storing it.
if errs := apivalidation.ValidateActorTemplateUpdate(ctx, field.NewPath("actor_template"), outTemplate, inTemplate); len(errs) > 0 {
return nil, toGRPCInternalError(errs)
}
// Reject a template whose actors could never be snapshotted, rather than
// failing their first suspend or tag.
if _, err := resolveSnapshotLocation(s.clusterSnapshotLocation, outTemplate); err != nil {
return nil, err
}
return s.store.CreateActorTemplate(ctx, outTemplate)
}
func (s *RPCService) GetActorTemplate(ctx context.Context, req *ateapipb.GetActorTemplateRequest) (*ateapipb.ActorTemplate, error) {
if errs := apivalidation.ValidateGetActorTemplateRequest(ctx, req); len(errs) > 0 {
return nil, resources.ToAPIError(errs)
}
templateRef := resources.ActorTemplateRefFromObjectRef(req.GetActorTemplate())
template, err := s.impl.GetActorTemplate(ctx, templateRef)
if errors.Is(err, store.ErrNotFound) {
return nil, apierror.NotFound("ActorTemplate %s not found", templateRef)
} else if err != nil {
return nil, fmt.Errorf("while getting actor template from DB: %w", err)
}
return template, nil
}
func (s *ServiceImpl) GetActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef) (*ateapipb.ActorTemplate, error) {
// TODO: implement this
return s.store.GetActorTemplate(ctx, templateRef)
}
func (s *RPCService) ListActorTemplates(ctx context.Context, req *ateapipb.ListActorTemplatesRequest) (*ateapipb.ListActorTemplatesResponse, error) {
if errs := apivalidation.ValidateListActorTemplatesRequest(ctx, req); len(errs) > 0 {
return nil, resources.ToAPIError(errs)
}
page, err := s.impl.ListActorTemplates(ctx, req.GetAtespace(), toStoreListOptions(req.GetOptions()))
if err != nil {
return nil, mapListError(fmt.Errorf("while listing actor templates in db: %w", err))
}
return &ateapipb.ListActorTemplatesResponse{
ActorTemplates: page.Items,
NextPageToken: page.NextPageToken,
}, nil
}
func (s *ServiceImpl) ListActorTemplates(ctx context.Context, atespace string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorTemplate], error) {
// TODO: implement this
return s.store.ListActorTemplates(ctx, atespace, opts)
}
func (s *RPCService) DeleteActorTemplate(ctx context.Context, req *ateapipb.DeleteActorTemplateRequest) (*ateapipb.ActorTemplate, error) {
if errs := apivalidation.ValidateDeleteActorTemplateRequest(ctx, req); len(errs) > 0 {
return nil, resources.ToAPIError(errs)
}
return s.actorWorkflow.DeleteActorTemplate(ctx, resources.ActorTemplateRefFromObjectRef(req.GetActorTemplate()), toDeletePreconditions(req.GetOptions()))
}
func (s *ServiceImpl) DeleteActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.DeletePreconditions) (*ateapipb.ActorTemplate, error) {
// TODO: implement this
return s.store.DeleteActorTemplate(ctx, templateRef, precondition)
}
func (s *ServiceImpl) UpdateActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef, precondition store.Precondition, mutate func(dbTemplate *ateapipb.ActorTemplate) error) (*ateapipb.ActorTemplate, error) {
// ActorTemplates are immutable to clients: there is no update RPC, and
// the only writer is the template reconciler, which updates status
// against the store directly. The store enforces metadata immutability,
// so this layer has nothing to add.
return s.store.UpdateActorTemplate(ctx, templateRef, precondition, mutate)
}
// actorTemplateGetter is the storage subset template resolution needs.
type actorTemplateGetter interface {
GetActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef) (*ateapipb.ActorTemplate, error)
}
// errActorTemplateNotFound matches (via errors.Is) resolution failures where
// the actor names a template that does not exist. Most callers return the
// error as is — it already carries FailedPrecondition — while delete
// tolerates it and cleans up without the template.
var errActorTemplateNotFound = apierror.FailedPrecondition("actor template not found")
// resolveActorTemplate resolves the substrate ActorTemplate the actor's
// actor_template ref names. A missing template surfaces as
// errActorTemplateNotFound.
func resolveActorTemplate(ctx context.Context, st actorTemplateGetter, actor *ateapipb.Actor) (*ateapipb.ActorTemplate, error) {
templateRef := resources.ActorTemplateRefFromObjectRef(actor.GetActorTemplate())
template, err := st.GetActorTemplate(ctx, templateRef)
if errors.Is(err, store.ErrNotFound) {
return nil, fmt.Errorf("%w; ObjectRef: %s ", errActorTemplateNotFound, templateRef)
}
if err != nil {
return nil, fmt.Errorf("while getting ActorTemplate: %w", err)
}
return template, nil
}