mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-10-02 03:24:42 +08:00
Steven Shriver
52c6a03c69
This change addresses six ways CI pipeline could report false success or hang on broken infrastructure. Importantly, this change does two things to reduce load on infrastructure: 1. It prevents jobs from running for [the default action limit of 360m](https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idtimeout-minutes) by pinning the timeout to `45m`. 2. It limits pull_request jobs from continuing execution once superseded by new changes. - **Silent container test skipping**: Tests now explicitly fail if `CI` or `REQUIRE_DOCKER` is set (while still skipping on local machines lacking Docker), with the check implemented in `dockerenv` to avoid an import cycle between `storetest` and `atepg`. - **Unbounded trust bundle wait**: Enforced a shared 120-second timeout across both bundles (overridable via `ATE_INSTALL_TRUST_BUNDLE_TIMEOUT`) and added diagnostic dumping of bundles, controller pods, and logs before returning a non-zero exit code. - **Missing sandbox preflight validation**: Added early preflight checks for `/dev/kvm` and `SandboxConfig/microvm` that fail fast and print actionable remediation instructions. - **Skipped migration checks on main**: Configured the migration immutability check to run on pushes to main to catch modified migrations at the point of merge. - **Missing job timeouts and concurrency limits**: Defined explicit timeout-minutes (45m and 120m bounds) and added concurrency groups that automatically cancel superseded pull request runs without canceling runs on main. Fixes #1747 - [x] Tests pass - Silent container skipping, unbounded trust bundles, and missing sandbox were all forced locally and confirmed to exist with changes here resolving each. - The latter half of the scenarios exist in CI only due to being GH Action trigger issues.
78 lines
2.6 KiB
Go
78 lines
2.6 KiB
Go
// Copyright 2026 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package e2e
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
|
|
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
|
|
)
|
|
|
|
// PreflightChecks checks that the test environment is ready for the test suite.
|
|
func PreflightChecks() error {
|
|
ctx := context.Background()
|
|
|
|
clients := GetClients()
|
|
|
|
// List namespaces to verify connectivity
|
|
_, err := clients.K8s.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to connect to Kubernetes API server: %v", err)
|
|
}
|
|
|
|
// Check deployments.
|
|
deployments := []string{
|
|
ResourceName("ate-controller"),
|
|
ResourceName("ate-api-server"),
|
|
}
|
|
namespace := SystemNamespace()
|
|
for _, depName := range deployments {
|
|
dep, err := clients.K8s.AppsV1().Deployments(namespace).Get(ctx, depName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("deployment %s/%s is missing: %v", namespace, depName, err)
|
|
}
|
|
if dep.Status.ReadyReplicas == 0 {
|
|
return fmt.Errorf("deployment %s/%s has 0 ready replicas. Status: %+v", namespace, depName, dep.Status)
|
|
}
|
|
}
|
|
|
|
// Verify that we can call the API.
|
|
listCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
|
|
_, err = clients.SubstrateAPI.ListActors(listCtx, &ateapipb.ListActorsRequest{})
|
|
if err != nil {
|
|
return fmt.Errorf("ListActors RPC failed: %v", err)
|
|
}
|
|
|
|
// The micro-VM class needs its SandboxConfig registered; without it every
|
|
// fixture in the run is misconfigured.
|
|
// Note: Whether a node can actually host a micro-VM is left to the scheduler
|
|
// and any issues are to be reported at runtime. Which device a sandbox class
|
|
// requires should not be checked here to prevent drift between platforms.
|
|
if IsMicroVM() {
|
|
if _, err := clients.SubstrateK8s.ApiV1alpha1().SandboxConfigs().Get(ctx, SandboxClassMicroVM, metav1.GetOptions{}); err != nil {
|
|
return fmt.Errorf("E2E_SANDBOX_CLASS=%s but SandboxConfig/%s is missing (see docs/dev/microvm-local.md for cluster setup): %w",
|
|
SandboxClassMicroVM, SandboxClassMicroVM, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|