rustfs/.github/workflows/nightly-gnu.yml

# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Nightly GNU Build

on:
  schedule:
    - cron: "7 0 * * *"
      timezone: "Asia/Shanghai"
  workflow_dispatch:
    inputs:
      branch:
        description: 'Branch/ref to build and publish as the nightly (empty = scheduled source, see NIGHTLY_BUILD_REF)'
        required: false
        default: ''

permissions:
  contents: read

# Scheduled builds follow the NIGHTLY_BRANCH repo variable so the channel can
# be pointed at e.g. `release` for the GA cycle and back to `main` afterwards
# without touching this file. Manual runs take the `branch` input, falling
# back to the branch the run was dispatched from.
concurrency:
  group: nightly-gnu-build-${{ github.event_name }}-${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}
  cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  NIGHTLY_BUILD_REF: ${{ github.event_name == 'schedule' && (vars.NIGHTLY_BRANCH || 'main') || (inputs.branch || github.ref_name) }}

jobs:
  resolve-source:
    if: github.repository == 'rustfs/rustfs'
    name: Resolve nightly source
    runs-on: sm-standard-2
    timeout-minutes: 10
    outputs:
      source_sha: ${{ steps.source.outputs.sha }}
      source_ref: ${{ steps.source.outputs.ref }}
    steps:
      - name: Checkout selected source
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
          ref: ${{ env.NIGHTLY_BUILD_REF }}
      - name: Record immutable source
        id: source
        run: |
          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
          echo "ref=${NIGHTLY_BUILD_REF}" >> "$GITHUB_OUTPUT"

  build:
    needs: resolve-source
    name: Build x86_64 GNU
    runs-on: sm-standard-4
    timeout-minutes: 150
    env:
      NIGHTLY_BUILD_REF: ${{ needs.resolve-source.outputs.source_ref }}
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
          ref: ${{ needs.resolve-source.outputs.source_sha }}

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          cache-shared-key: build-x86_64-unknown-linux-gnu-glibc-2.31
          cache-save-if: 'false'
          install-cross-tools: 'true'
          target: x86_64-unknown-linux-gnu
          install-build-packaging-tools: 'false'
          install-test-tools: 'false'

      - name: Build RustFS
        # Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
        # nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
        run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins

      - name: Verify glibc symbol baseline
        run: |
          python3 scripts/release/check_glibc.py 2.31 \
            target/x86_64-unknown-linux-gnu/release/rustfs \
            target/x86_64-unknown-linux-gnu/release/rustfs-cli
          target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'

      - name: Build DEB package
        id: deb
        shell: bash
        run: |
          set -euo pipefail

          # Nightly snapshot name: rustfs-nightly-<YYYY-MM-DD> (Asia/Shanghai,
          # matching the schedule timezone so the file date always matches the
          # cron's intended day).
          DEB_DATE="$(TZ=Asia/Shanghai date +%Y-%m-%d)"
          DEB_FILE="rustfs-nightly-${DEB_DATE}.deb"
          PKG_DIR="rustfs-nightly-${DEB_DATE}"

          command -v fakeroot >/dev/null 2>&1 || sudo apt-get install -y -qq fakeroot

          BIN="target/x86_64-unknown-linux-gnu/release/rustfs"
          test -x "${BIN}" || { echo "rustfs binary not found: ${BIN}"; exit 1; }

          mkdir -p "${PKG_DIR}/DEBIAN"
          mkdir -p "${PKG_DIR}/usr/bin"
          mkdir -p "${PKG_DIR}/etc/default"
          mkdir -p "${PKG_DIR}/lib/systemd/system"
          mkdir -p "${PKG_DIR}/usr/share/doc/rustfs"

          cp "${BIN}" "${PKG_DIR}/usr/bin/rustfs"
          chmod 755 "${PKG_DIR}/usr/bin/rustfs"
          cp deploy/build/rustfs.service "${PKG_DIR}/lib/systemd/system/"

          cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
          # RustFS Environment Configuration
          # See https://docs.rustfs.com for more information
          # RUSTFS_ACCESS_KEY=<changeme>
          # RUSTFS_SECRET_KEY=<changeme>
          # RUSTFS_VOLUMES=""
          # RUSTFS_ADDRESS=":9000"
          # RUSTFS_CONSOLE_ADDRESS=":9001"
          # RUSTFS_CONSOLE_ENABLE=true
          # RUSTFS_OBS_LOGGER_LEVEL=error
          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
          ENVEOF

          # dpkg versions must start with a digit and cannot contain hyphens;
          # a date-based snapshot version keeps the nightly installable
          # alongside release packages.
          DEB_VERSION="${DEB_DATE//-/.}~nightly"

          cat > "${PKG_DIR}/DEBIAN/control" << EOF
          Package: rustfs
          Version: ${DEB_VERSION}
          Section: utils
          Priority: optional
          Architecture: amd64
          Depends: libc6 (>= 2.31)
          Maintainer: RustFS Team <support@rustfs.com>
          Description: High-performance distributed object storage
           RustFS is a high-performance distributed object storage software
           built using Rust. It is compatible with MinIO and S3 API.
          Homepage: https://rustfs.com
          EOF

          cat > "${PKG_DIR}/DEBIAN/conffiles" << 'CONFFILES'
          /etc/default/rustfs
          CONFFILES

          ./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
          chmod 755 "${PKG_DIR}/DEBIAN/postinst"

          ./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
          chmod 755 "${PKG_DIR}/DEBIAN/prerm"

          ./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
          chmod 755 "${PKG_DIR}/DEBIAN/postrm"

          cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
          cp README.md "${PKG_DIR}/usr/share/doc/rustfs/"

          fakeroot dpkg-deb --build "${PKG_DIR}"
          ls -lh "${DEB_FILE}"
          echo "deb_date=${DEB_DATE}" >> "${GITHUB_OUTPUT}"
          echo "deb_file=${DEB_FILE}" >> "${GITHUB_OUTPUT}"

      # Same packaging scheme as .github/workflows/package.yml (fpm), but from
      # the locally built nightly binary instead of a release artifact, with a
      # date-based version that mirrors the DEB.
      - name: Build RPM package
        id: rpm
        shell: bash
        env:
          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
        run: |
          set -euo pipefail

          if ! command -v fpm >/dev/null 2>&1; then
            SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
            ${SUDO} apt-get update -qq && ${SUDO} apt-get install -y -qq ruby ruby-dev build-essential rpm >/dev/null
            ${SUDO} gem install fpm --no-document >/dev/null
          fi

          RPM_FILE="rustfs-nightly-${DEB_DATE}.rpm"
          RPM_VERSION="0"
          RPM_RELEASE="0.nightly.${DEB_DATE//-/.}"

          echo "Building RPM: ${RPM_FILE} (version ${RPM_VERSION}-${RPM_RELEASE})"

          # fpm wants the config file to exist before packaging.
          mkdir -p ./tmp-pkg/etc/default
          cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
          # RustFS Environment Configuration
          # See https://docs.rustfs.com for more information
          # RUSTFS_ACCESS_KEY=<changeme>
          # RUSTFS_SECRET_KEY=<changeme>
          # RUSTFS_VOLUMES=""
          # RUSTFS_ADDRESS=":9000"
          # RUSTFS_CONSOLE_ADDRESS=":9001"
          # RUSTFS_CONSOLE_ENABLE=true
          # RUSTFS_OBS_LOGGER_LEVEL=error
          # RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
          ENVEOF

          fpm -s dir -t rpm \
            --name rustfs \
            --version "$RPM_VERSION" \
            --iteration "$RPM_RELEASE" \
            --architecture x86_64 \
            --package "$RPM_FILE" \
            --depends "glibc >= 2.31" \
            --maintainer "RustFS Team <support@rustfs.com>" \
            --description "High-performance distributed object storage" \
            --url "https://rustfs.com" \
            --license "Apache-2.0" \
            --before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
            --before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
            --after-install <(./scripts/release/package_service_scripts.sh after-install) \
            --after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
            --before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
            --after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
            --rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
            --config-files /etc/default/rustfs \
            "rustfs-nightly-${DEB_DATE}/usr/bin/rustfs=/usr/bin/rustfs" \
            ./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
            deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
            LICENSE=/usr/share/doc/rustfs/LICENSE \
            README.md=/usr/share/doc/rustfs/README.md

          [[ -f "$RPM_FILE" ]] || { echo "RPM build failed"; exit 1; }
          rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
          stat --printf='%n %s bytes\n' "$RPM_FILE"
          echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"

      - name: Upload DEB artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ steps.deb.outputs.deb_file }}
          path: ${{ steps.deb.outputs.deb_file }}
      - name: Upload RPM artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ steps.rpm.outputs.rpm_file }}
          path: ${{ steps.rpm.outputs.rpm_file }}
          if-no-files-found: error

      # Persist the nightly deb on Cloudflare R2 (same channel as package.yml)
      # so it can be downloaded later with a stable, unauthenticated URL —
      # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... .
      # Skipped when the R2 secrets are not configured (artifact-only mode).
      - name: Upload DEB to Cloudflare R2
        id: publish
        env:
          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
          R2_BUCKET: ${{ secrets.R2_BUCKET }}
          AWS_EC2_METADATA_DISABLED: true
        shell: bash
        run: |
          set -euo pipefail

          if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
            echo "⚠️ R2 credentials missing, skipping upload"
            exit 0
          fi

          export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
          export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
          export AWS_DEFAULT_REGION="auto"

          # The candidate manifest must describe the tree that was actually
          # built. With a ref override (NIGHTLY_BRANCH / dispatch input) that
          # is not necessarily GITHUB_SHA, so always advertise HEAD.
          SOURCE_SHA="$(git rev-parse HEAD)"
          DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)"
          CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb"
          CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}"

          # Old AWS CLI models lack conditional PutObject support. Never fall
          # back to an overwriting upload for a candidate.
          AWS_CLI=aws
          if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
            sudo apt-get update
            sudo apt-get install -y -qq python3-venv
            AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")"
            trap 'rm -rf "${AWS_CLI_DIR}"' EXIT
            python3 -m venv "${AWS_CLI_DIR}"
            "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
            AWS_CLI="${AWS_CLI_DIR}/bin/aws"
          fi
          "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
          "${AWS_CLI}" --version
          "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \
            --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
          PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)"
          if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then
            echo "Published candidate checksum does not match the built package" >&2
            exit 1
          fi

          R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/"

          echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}"
          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors

          # Stable "latest" alias so tests can fetch the newest nightly
          # without knowing today's date.
          echo "📤 Uploading latest alias"
          "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \
            --endpoint-url "$R2_ENDPOINT" --only-show-errors

          echo "✅ R2 upload complete"

          CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
          jq -n --arg source_sha "${SOURCE_SHA}" \
            --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${NIGHTLY_BUILD_REF}" \
            --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
            --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \
            '{schema: 2, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \
            > "${CANDIDATE_FILE}"
          echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}"

      - name: Upload nightly candidate manifest
        if: ${{ steps.publish.outputs.candidate_file != '' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}
          path: ${{ steps.publish.outputs.candidate_file }}
          if-no-files-found: error

      # Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
      # engineers can download and install the nightly directly. The branch is
      # a single-commit orphan rewritten on every build, which keeps the repo
      # small while the latest files stay reachable at stable raw URLs.
      # Publish the deb/rpm pair as assets of the rolling `nightly` release on
      # rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
      - name: Publish packages to auto-testing release assets
        env:
          ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
          DEB_FILE: ${{ steps.deb.outputs.deb_file }}
          RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
          DEB_DATE: ${{ steps.deb.outputs.deb_date }}
          BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
        run: bash scripts/release/publish_nightly_assets.sh
  # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
  #
  # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
  # Vault Transit backends to every for_each_backend spec in
  # crates/kms/tests/behavior_*.rs (see crates/kms/AGENTS.md). rotate and
  # versioning are advertised only by the Vault backends, so without this lane
  # no CI run ever asserts the working half of behavior_rotation.rs — a
  # rotation that silently dropped historical key versions would stay green.
  # The same lane runs the dev-Vault #[ignore] tests and the two self-hosting
  # live scripts (AppRole login, three-node Raft leader failover).
  #
  # GitHub-hosted ubuntu-latest, deliberately not the self-hosted sm-standard
  # fleet: both lanes need a working Docker daemon, and the sm-standard-2 pool
  # does not provide one. #8112 rerouted these lanes there and both died in
  # seconds on "Docker is not available" while the package itself built and
  # published fine, which kept the nightly chain from firing (its gate
  # requires a successful build run). See also the banner in e2e-s3tests.yml
  # and rustfs/backlog#1149.
  kms-vault-lane:
    needs: resolve-source
    name: KMS live Vault lane
    runs-on: ubuntu-latest
    timeout-minutes: 90
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      # Root token of the ephemeral loopback dev server. Not a secret: the
      # server lives only for this job, listens on 127.0.0.1, and holds only
      # keys the tests create. The literal value matters — the dev-Vault
      # #[ignore] fixtures in crates/kms/src/backends/vault.rs hardcode it.
      VAULT_LANE_TOKEN: dev-only-token
      VAULT_LANE_ADDR: http://127.0.0.1:8200
      # Keeps a runner-level proxy from swallowing the loopback dev-server
      # traffic (see crates/kms/AGENTS.md). Actions env keys are
      # case-insensitive, so only the uppercase form is set; reqwest reads
      # either casing.
      NO_PROXY: 127.0.0.1,localhost
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
          ref: ${{ needs.resolve-source.outputs.source_sha }}

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          # Dedicated key: rust-cache cannot tell runner images apart, so
          # sharing a key with an sm-standard lane would let two different
          # system images overwrite each other's artifacts (same reasoning as
          # ci.yml's ci-uring lane). Saved from this nightly job itself so the
          # next night starts warm.
          cache-shared-key: kms-vault-lane
          cache-save-if: 'true'
          install-build-packaging-tools: 'false'
          install-test-tools: 'false'

      - name: Install Vault CLI
        run: |
          set -euo pipefail
          wget -qO- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list >/dev/null
          sudo apt-get update -qq
          sudo apt-get install -y -qq vault
          vault version

      - name: Start Vault dev server with KV2 and Transit engines
        run: |
          set -euo pipefail
          nohup vault server -dev \
            -dev-root-token-id="${VAULT_LANE_TOKEN}" \
            -dev-listen-address=127.0.0.1:8200 >/tmp/vault-dev.log 2>&1 &
          for _ in $(seq 1 60); do
            if curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health" >/dev/null 2>&1; then
              break
            fi
            sleep 1
          done
          curl -fsS "${VAULT_LANE_ADDR}/v1/sys/health"
          export VAULT_ADDR="${VAULT_LANE_ADDR}" VAULT_TOKEN="${VAULT_LANE_TOKEN}"
          # Dev mode mounts KV v2 at secret/ by default; Transit is explicit.
          # Prove both engines actually work rather than assuming the defaults.
          vault secrets enable transit
          vault kv put secret/rustfs-ci-lane-probe value=ok >/dev/null
          vault kv get secret/rustfs-ci-lane-probe >/dev/null
          vault write -f transit/keys/rustfs-ci-lane-probe >/dev/null

      - name: Run rustfs-kms suite with the Vault lane on
        env:
          RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
          RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
        run: cargo test -p rustfs-kms --locked

      - name: Run dev-Vault ignored tests
        env:
          RUSTFS_KMS_VAULT_TOKEN: ${{ env.VAULT_LANE_TOKEN }}
          RUSTFS_KMS_VAULT_ADDR: ${{ env.VAULT_LANE_ADDR }}
        # Filters select the dev-Vault-only #[ignore] tests. The AWS #[ignore]
        # tests (backends::aws, service_manager) stay excluded — they need real
        # AWS credentials and create billable keys. The AppRole and HA #[ignore]
        # tests are excluded here because their own scripts below provision the
        # Vault topology they need.
        run: |
          set -euo pipefail
          cargo test -p rustfs-kms --locked --lib backends::contract_tests -- --ignored
          cargo test -p rustfs-kms --locked --lib backends::vault -- --ignored
          cargo test -p rustfs-kms --locked --test vault_fault_injection -- --ignored

      - name: Run AppRole live checks (self-hosting ephemeral Vault)
        run: bash scripts/test/vault_approle_kms_live.sh

      - name: Show Vault dev server log on failure
        if: failure()
        run: tail -n 200 /tmp/vault-dev.log || true

  # Three-node Raft leader failover (crates/kms/tests/vault_ha_failover_live.rs,
  # first validated by rustfs/rustfs#5653). Its own job so an election-timing
  # flake cannot mask the main lane's verdict, and vice versa. The script
  # provisions and tears down its own Docker cluster.
  kms-vault-ha-failover:
    needs: resolve-source
    name: KMS Vault HA failover lane
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
      NO_PROXY: 127.0.0.1,localhost
    steps:
      - name: Checkout repository
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
          ref: ${{ needs.resolve-source.outputs.source_sha }}

      - name: Setup Rust environment
        uses: ./.github/actions/setup
        with:
          cache-shared-key: kms-vault-lane
          cache-save-if: 'false'
          install-build-packaging-tools: 'false'
          install-test-tools: 'false'

      - name: Run HA leader failover live checks (three-node Raft cluster in Docker)
        run: bash scripts/test/vault_ha_kms_live.sh

  alert-on-failure:
    name: Alert on scheduled failure
    needs: [build, kms-vault-lane, kms-vault-ha-failover]
    if: >-
      github.repository == 'rustfs/rustfs' && (
        always() && github.event_name == 'schedule' &&
        (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
      )
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
        with:
          persist-credentials: false
      - name: Open or update failure-tracking issue
        uses: ./.github/actions/schedule-failure-issue
        with:
          github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}