mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-11-12 10:35:06 +08:00
Zoe Zhao
f6852b7754
This PR is very large since it updates all existing demos and benchmark workloads to use the new ActorTemplate substrate proto. Please use the "Commits" tab to review individual commits. Verifications done: * Used this script: gpaste/5143788763348992 to verify that the change from CRD -> proto are equivalent. * The e2e tests are using the new susbtrate resources. * Picked the parking demo to run e2e manually: gpaste/6193361380311040
313 lines
11 KiB
Bash
Executable File
313 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
set -o errexit -o nounset -o pipefail
|
|
|
|
ROOT="$(git rev-parse --show-toplevel)"
|
|
cd "${ROOT}"
|
|
|
|
# Source the environment variables if configured
|
|
if [[ -f .ate-dev-env.sh ]]; then
|
|
source .ate-dev-env.sh
|
|
fi
|
|
|
|
# Ensure BUCKET_NAME is set
|
|
if [[ -z "${BUCKET_NAME:-}" ]]; then
|
|
echo "Error: BUCKET_NAME environment variable is not set." >&2
|
|
exit 1
|
|
fi
|
|
|
|
MANIFEST_DIR="benchmarking/workloads/manifests"
|
|
POOL_MANIFEST="${MANIFEST_DIR}/workloads.yaml.tmpl"
|
|
# The benchmark ActorTemplates: <name>-template.yaml.tmpl each, created
|
|
# through the ate API in the benchmark-workloads atespace. WORKLOAD_TEMPLATES
|
|
# overrides the default set — the usermem and kernelmem templates (for the
|
|
# matching locust tests) are not deployed by default.
|
|
read -r -a TEMPLATES <<<"${WORKLOAD_TEMPLATES:-sleep glutton glutton-durdir-data glutton-durdir-full}"
|
|
|
|
if [[ ! -f "${POOL_MANIFEST}" ]]; then
|
|
echo "Error: ${POOL_MANIFEST} not found in $(pwd)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
WORKER_COUNT=1
|
|
SANDBOX_CLASS="gvisor"
|
|
# Actor memory limit (ActorTemplate spec.resources.limits.memory). The default
|
|
# is the smallest size microvm admits (128Mi VMM reserve + 128Mi guest floor),
|
|
# so benchmark actors do not inherit the 2 GiB kata default and drag its page
|
|
# cache into every memory snapshot. Raise it for RAM-consuming suites.
|
|
ACTOR_MEMORY="256Mi"
|
|
# The address to which an instrumented actor container sends its telemetry.
|
|
# --otlp-endpoint sets it. Without the flag, resolve_otlp_endpoint reads the
|
|
# address that the control plane uses.
|
|
OTLP_ENDPOINT=""
|
|
# The timeout, in whole seconds, for waiting for the ateom worker pods to be
|
|
# ready.
|
|
WAIT_TIMEOUT_SECS=300
|
|
|
|
usage() {
|
|
echo "Usage: $0 [options]"
|
|
echo ""
|
|
echo "Options:"
|
|
echo " --deploy Substitute env vars and deploy workloads to the cluster using ko apply"
|
|
echo " --delete Substitute env vars and delete workloads from the cluster"
|
|
echo " --worker-count N Number of WorkerPool replicas (default: 1)"
|
|
echo " --sandbox-class CLASS Sandbox runtime for the WorkerPool: gvisor | microvm (default: gvisor)."
|
|
echo " microvm requires hack/install-microvm-deps.sh --install to have run."
|
|
echo " --actor-memory SIZE Memory limit for the benchmark ActorTemplates (default: 256Mi,"
|
|
echo " the smallest size microvm admits)"
|
|
echo " --otlp-endpoint URL The address to which an instrumented actor container"
|
|
echo " sends telemetry (default: the endpoint in the"
|
|
echo " ate-otel-config ConfigMap)"
|
|
echo " --wait-timeout SECONDS The timeout in seconds for waiting for the ateom workers to be ready (default: 300)"
|
|
echo " -h, --help Show this help message"
|
|
}
|
|
|
|
# Read the endpoint from the ate-otel-config ConfigMap, which every control
|
|
# plane component reads through envFrom. The value is correct for the cluster
|
|
# in use: the GKE collector, the kind collector in otel-system, or the
|
|
# telemetry meter while a measurement runs. Thus the actors follow the control
|
|
# plane, and this script needs no test for the type of cluster.
|
|
#
|
|
# ate-system must exist, because the workloads below need its CRDs. Thus an
|
|
# absent ConfigMap is an error and not a condition to work around: a default
|
|
# here would send the actor telemetry to the wrong collector with no message.
|
|
resolve_otlp_endpoint() {
|
|
if [[ -n "${OTLP_ENDPOINT}" ]]; then
|
|
return 0
|
|
fi
|
|
OTLP_ENDPOINT="$(kubectl get configmap ate-otel-config --namespace=ate-system \
|
|
-o jsonpath='{.data.OTEL_EXPORTER_OTLP_ENDPOINT}' 2>/dev/null || true)"
|
|
if [[ -z "${OTLP_ENDPOINT}" ]]; then
|
|
echo "Error: cannot read OTEL_EXPORTER_OTLP_ENDPOINT from the ate-otel-config" >&2
|
|
echo "ConfigMap in ate-system. Deploy ate-system first, or give --otlp-endpoint." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# kubectl-ate runs once per poll while waiting for the golden snapshots;
|
|
# build_kubectl_ate builds it once up front instead of paying the `go run`
|
|
# toolchain startup on every call.
|
|
KUBECTL_ATE_BIN=""
|
|
|
|
build_kubectl_ate() {
|
|
local dir
|
|
dir="$(mktemp -d)"
|
|
trap 'rm -rf '"${dir}" EXIT
|
|
KUBECTL_ATE_BIN="${dir}/kubectl-ate"
|
|
go build -o "${KUBECTL_ATE_BIN}" ./cmd/kubectl-ate
|
|
}
|
|
|
|
run_kubectl_ate() {
|
|
"${KUBECTL_ATE_BIN}" "$@"
|
|
}
|
|
|
|
substitute() {
|
|
# SandboxConfig names are pinned per class (rather than defaulted) so a stale
|
|
# config from a dirty teardown fails loudly instead of silently binding this
|
|
# pool. gvisor-default is applied by hack/install-ate.sh; microvm is applied
|
|
# by hack/install-microvm-deps.sh. The protojson templates take the sandbox
|
|
# class as its proto enum spelling.
|
|
local manifest="$1"
|
|
local sandbox_config_name sandbox_class_enum
|
|
case "${SANDBOX_CLASS}" in
|
|
gvisor) sandbox_config_name="gvisor-default" sandbox_class_enum="SANDBOX_CLASS_GVISOR" ;;
|
|
microvm) sandbox_config_name="microvm" sandbox_class_enum="SANDBOX_CLASS_MICROVM" ;;
|
|
esac
|
|
sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \
|
|
-e "s|\${WORKER_COUNT}|${WORKER_COUNT}|g" \
|
|
-e "s|\${SANDBOX_CLASS}|${SANDBOX_CLASS}|g" \
|
|
-e "s|\${SANDBOX_CLASS_ENUM}|${sandbox_class_enum}|g" \
|
|
-e "s|\${SANDBOX_CONFIG_NAME}|${sandbox_config_name}|g" \
|
|
-e "s|\${OTLP_ENDPOINT}|${OTLP_ENDPOINT}|g" \
|
|
-e "s|\${ACTOR_MEMORY}|${ACTOR_MEMORY}|g" \
|
|
"${manifest}"
|
|
}
|
|
|
|
# wait_actortemplate_ready polls a substrate ActorTemplate resource until its
|
|
# golden snapshot exists (the substrate counterpart of `kubectl wait
|
|
# --for=condition=Ready actortemplate/...`). Fails fast when the template
|
|
# reconciler reports an error.
|
|
wait_actortemplate_ready() {
|
|
local atespace="$1"
|
|
local template="$2"
|
|
local timeout_secs="${3:-300}"
|
|
local deadline=$((SECONDS + timeout_secs))
|
|
local json snapshot error_message
|
|
|
|
while ((SECONDS < deadline)); do
|
|
if json=$(run_kubectl_ate get actor-template "${template}" -a "${atespace}" -o json 2>/dev/null); then
|
|
snapshot=$(jq -r '.actorTemplates[0].status.goldenSnapshotStatus.goldenSnapshot.name // empty' <<<"${json}")
|
|
if [[ -n "${snapshot}" ]]; then
|
|
return 0
|
|
fi
|
|
error_message=$(jq -r '.actorTemplates[0].status.goldenSnapshotStatus.errorMessage // empty' <<<"${json}")
|
|
if [[ -n "${error_message}" ]]; then
|
|
echo "actor template ${atespace}/${template} failed: ${error_message}" >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
echo "timed out waiting for actor template ${atespace}/${template} golden snapshot" >&2
|
|
return 1
|
|
}
|
|
|
|
# wait_templates_ready blocks until every benchmark template's golden
|
|
# snapshot exists (there is no kubectl wait for substrate resources), failing
|
|
# fast when the template reconciler reports an error.
|
|
wait_templates_ready() {
|
|
if ! command -v jq &>/dev/null; then
|
|
echo "jq is required to wait for the benchmark actor templates" >&2
|
|
return 1
|
|
fi
|
|
local template
|
|
for template in "${TEMPLATES[@]}"; do
|
|
echo "Waiting for the benchmark-workloads/${template} golden snapshot..."
|
|
wait_actortemplate_ready benchmark-workloads "${template}" "${WAIT_TIMEOUT_SECS}"
|
|
done
|
|
}
|
|
|
|
deploy() {
|
|
resolve_otlp_endpoint
|
|
echo "Deploying workloads (worker_count=${WORKER_COUNT}, actor_memory=${ACTOR_MEMORY}, otlp_endpoint=${OTLP_ENDPOINT})..."
|
|
substitute "${POOL_MANIFEST}" | hack/run-tool.sh ko apply -f -
|
|
echo "Waiting for worker pool to be ready (timeout: ${WAIT_TIMEOUT_SECS}s)..."
|
|
kubectl wait --for=create deployment/benchmark-ateom \
|
|
--namespace=benchmark-workloads --timeout="${WAIT_TIMEOUT_SECS}s"
|
|
kubectl rollout status deployment/benchmark-ateom \
|
|
--namespace=benchmark-workloads --timeout="${WAIT_TIMEOUT_SECS}s"
|
|
|
|
# The store enforces that a template's atespace exists at create time.
|
|
run_kubectl_ate create atespace benchmark-workloads >/dev/null 2>&1 \
|
|
|| run_kubectl_ate get atespace benchmark-workloads >/dev/null
|
|
|
|
# Actor templates are immutable (no update RPC). A value that changes for
|
|
# each run — the OTLP endpoint or the sandbox class — needs the removal of
|
|
# the old template first. This removal is safe, because the benchmark
|
|
# automation deletes the actors between the tests; it also removes the old
|
|
# golden actor and snapshot server-side.
|
|
local template
|
|
for template in "${TEMPLATES[@]}"; do
|
|
run_kubectl_ate delete actor-template "${template}" -a benchmark-workloads \
|
|
>/dev/null 2>&1 || true
|
|
# ko resolve builds the ko:// image references and replaces them with
|
|
# pushed digests before the manifest reaches kubectl-ate.
|
|
substitute "${MANIFEST_DIR}/${template}-template.yaml.tmpl" \
|
|
| hack/run-tool.sh ko resolve -f - \
|
|
| run_kubectl_ate create actor-template -f -
|
|
done
|
|
|
|
wait_templates_ready
|
|
}
|
|
|
|
delete() {
|
|
echo "Deleting workloads..."
|
|
local template
|
|
for template in "${TEMPLATES[@]}"; do
|
|
run_kubectl_ate delete actor-template "${template}" -a benchmark-workloads \
|
|
>/dev/null 2>&1 || true
|
|
done
|
|
run_kubectl_ate delete atespace benchmark-workloads >/dev/null 2>&1 \
|
|
|| echo "atespace benchmark-workloads not deleted (may not exist or is not empty)"
|
|
# The pool manifest contains ko:// image references; route through
|
|
# `ko delete` so they get resolved before kubectl sees them.
|
|
substitute "${POOL_MANIFEST}" | hack/run-tool.sh ko delete --ignore-not-found -f -
|
|
}
|
|
|
|
if [[ "$#" -eq 0 ]]; then
|
|
usage
|
|
exit 1
|
|
fi
|
|
|
|
action=""
|
|
while [[ "$#" -gt 0 ]]; do
|
|
case "$1" in
|
|
--deploy)
|
|
action="deploy"
|
|
;;
|
|
--delete)
|
|
action="delete"
|
|
;;
|
|
--worker-count)
|
|
shift
|
|
WORKER_COUNT="$1"
|
|
;;
|
|
--worker-count=*)
|
|
WORKER_COUNT="${1#*=}"
|
|
;;
|
|
--sandbox-class)
|
|
shift
|
|
SANDBOX_CLASS="$1"
|
|
;;
|
|
--sandbox-class=*)
|
|
SANDBOX_CLASS="${1#*=}"
|
|
;;
|
|
--otlp-endpoint)
|
|
shift
|
|
OTLP_ENDPOINT="$1"
|
|
;;
|
|
--otlp-endpoint=*)
|
|
OTLP_ENDPOINT="${1#*=}"
|
|
;;
|
|
--actor-memory)
|
|
shift
|
|
ACTOR_MEMORY="$1"
|
|
;;
|
|
--actor-memory=*)
|
|
ACTOR_MEMORY="${1#*=}"
|
|
;;
|
|
--wait-timeout)
|
|
shift
|
|
WAIT_TIMEOUT_SECS="$1"
|
|
;;
|
|
--wait-timeout=*)
|
|
WAIT_TIMEOUT_SECS="${1#*=}"
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Error: Unknown option: $1" >&2
|
|
usage
|
|
exit 1
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
case "${SANDBOX_CLASS}" in
|
|
gvisor|microvm) ;;
|
|
*)
|
|
echo "Error: --sandbox-class must be gvisor or microvm, got '${SANDBOX_CLASS}'" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if ! [[ "${WAIT_TIMEOUT_SECS}" =~ ^[0-9]+$ ]]; then
|
|
echo "Error: --wait-timeout must be a whole number of seconds like 300, got '${WAIT_TIMEOUT_SECS}'" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${action}" == "deploy" ]]; then
|
|
build_kubectl_ate
|
|
deploy
|
|
elif [[ "${action}" == "delete" ]]; then
|
|
build_kubectl_ate
|
|
delete
|
|
fi
|