#!/bin/sh
# Did the release that just booted actually come up? If not, hand over to `robot-rescue`.
#
# Started once per boot by `robot-boot-check.timer`, a second in, and it watches from there. See
# `docs/design/boot-recovery-net.md` for why this exists at all and why it watches rather than
# relying on `OnFailure=` on each daemon.
#
# WHEN IT DECIDES. Every couple of seconds it looks at the members, and stops at the first of:
#
#   - a member did not come up (below): hand over at once — counts only grow, so waiting longer
#     cannot change the answer;
#   - every member has been `active` for 10 s straight: the release came up. A crash-looper re-enters
#     `active` on every restart, so it never stays there long enough to count;
#   - the deadline, `[update_gate] boot_deadline_s` in robotd.toml (60 s): judge what is there. A
#     member still not settled by then — stopped by hand, or never started — is left alone, as below.
#
# WHAT COUNTS AS "DID NOT COME UP". Two conditions, both positive evidence that systemd tried to run
# a daemon and could not keep it running:
#
#   - the unit is `failed`;
#   - the unit has restarted at least 3 times this boot, whatever it happens to be doing right now.
#
# Everything else is left alone, and that asymmetry is the point. A unit an operator stopped by hand
# is `inactive` with no restarts, and must not cost the board its release. A unit a release predates
# is not loaded, which reads the same way. A daemon still waiting for hardware is `active` — all four
# members wait rather than exit, which is what makes them members at all — and a single crash that
# recovered leaves one or two restarts, not three. A false negative here costs an operator a
# diagnosis; a false positive costs a good release.
#
# WHY THESE FOUR. A unit may join only if it waits for its hardware rather than exiting, so that a
# unit which is down means the *binary* is broken — which is what a rollback fixes. `padd` is
# excluded because it exits cleanly when robotd's socket is absent, by design. `btd` is included on
# merit rather than symmetry: with no network, BLE is the only way in, so a robot that boots with no
# `btd` and no known wifi cannot be reached at all.
#
# Usage:  robot-boot-check [--dry-run]
#
# WHETHER IT ACTS. Only with `[update_gate] auto_rollback = true` in robotd.toml, which is off by
# default: a release that did not come up is then named in the journal and left where it is.
# `--dry-run` answers the same question either way, because the OS's slot gate reads it.
#
# Exit: 0 nothing to do, or the rescue was handed the decision; 2 declined; 3 with --dry-run, would
# have handed over; 1 something went wrong.
set -eu

MEMBERS="updaterd.service robotd.service configd.service btd.service"

# Three, not one. `Restart=always` plus a 2-5s `RestartSec` means a genuine loop passes three well
# inside the deadline, while a daemon that died once on a transient and came back does not. One would
# fire on the latter. `updaterd` is the slow one, `RestartSec=10s`: a loop there passes three about
# 45 s into a boot, which is what holds the deadline's default at 60.
MAX_RESTARTS=3

# How long every member must have been `active`, unbroken, to call the release up. Past the gap
# between two starts of every member's restart loop, so a crash-looper is never caught looking well.
SETTLE=10
POLL=2

# `[update_gate] boot_deadline_s`, and its bounds. `robotd-params` owns the key and repeats these as
# `BOOT_DEADLINE_*`; `xtask/tests/rescue.rs` keeps the two equal. Clamped rather than refused, here and
# there: every daemon reading robotd.toml exits on a value it refuses, and a robotd that will not start
# is exactly what this script rolls a release back for.
DEADLINE_DEFAULT=60
DEADLINE_MIN=20
DEADLINE_MAX=300

# This is a *boot* check. Being invoked well after boot means something other than the timer started
# it — a hand-typed `systemctl start`, or an installer enabling units — and the honest answer then is
# that the question is not this script's business. Without this guard, `hooks/postinstall` running
# `enable --now` over the units it installs would run a rollback check in the middle of an update,
# with daemons legitimately mid-restart.
MAX_UPTIME=600

RESCUE="${ROBOT_RESCUE:-/usr/local/sbin/robot-rescue}"
UPTIME_FILE="${ROBOT_UPTIME_FILE:-/proc/uptime}"
# robotd.toml's two layers, the owner's winning key by key (`robotd_params::USER_PATH`).
PARAMS_BASE="${ROBOT_PARAMS_BASE:-/etc/robot/robotd.toml}"
PARAMS_USER="${ROBOT_PARAMS_USER:-/var/lib/robot/robotd.toml}"

DRY_RUN=
while [ $# -gt 0 ]; do
    case "$1" in
        --dry-run) DRY_RUN=1 ;;
        -h|--help)
            cat <<'USAGE'
usage: robot-boot-check [--dry-run]

Decide whether the booted release brought its daemons up, and hand over to robot-rescue if not.
Started once per boot by robot-boot-check.timer. Hands over only with
[update_gate] auto_rollback = true in robotd.toml. See docs/design/boot-recovery-net.md.

  --dry-run   report the verdict, and do not invoke robot-rescue

Exit: 0 nothing to do or handed over, 2 declined, 3 would have handed over (--dry-run),
1 something went wrong.
USAGE
            exit 0
            ;;
        *)
            echo "robot-boot-check: unknown argument: $1" >&2
            exit 1
            ;;
    esac
    shift
done

say() {
    echo "robot-boot-check: $*" >&2
    command -v logger >/dev/null 2>&1 && logger -t robot-boot-check "$*"
    return 0
}

# Seconds since boot, or empty when there is nothing to read it from — which is not a reason to
# refuse to run, only a reason not to enforce the guard.
uptime_secs() {
    [ -r "$UPTIME_FILE" ] || return 0
    cut -d' ' -f1 < "$UPTIME_FILE" | cut -d. -f1
}

up="$(uptime_secs)"
if [ -n "$up" ] && [ "$up" -gt "$MAX_UPTIME" ]; then
    say "up for ${up}s, which is well past the boot deadline; this is not a boot check.
  Nothing done. To act anyway, decide for yourself and run: robot-rescue --dry-run"
    exit 2
fi

# One `[update_gate]` key from one layer of robotd.toml, or nothing. A line-level read of one key in
# one section rather than a TOML parser, because this runs when the release cannot: whatever it fails
# to read falls back to the default, which is the safe direction.
gate_key_in() {
    # $1 layer, $2 key
    [ -r "$1" ] || return 0
    awk -v key="$2" '
        /^[ \t]*\[/ { section = $0; sub(/#.*/, "", section); gsub(/[ \t]/, "", section); next }
        section == "[update_gate]" && $0 ~ ("^[ \t]*" key "[ \t]*=") {
            value = $0; sub(/^[^=]*=[ \t]*/, "", value); sub(/[ \t]*(#.*)?$/, "", value)
            print value; exit
        }
    ' "$1"
}

deadline="$DEADLINE_DEFAULT"
for layer in "$PARAMS_BASE" "$PARAMS_USER"; do
    value="$(gate_key_in "$layer" boot_deadline_s)"
    [ -n "$value" ] || continue
    case "$value" in
        *[!0-9]*)
            say "${layer}: boot_deadline_s = ${value} is not a whole number of seconds; ignoring it"
            continue
            ;;
    esac
    deadline="$value"
done
if [ "$deadline" -lt "$DEADLINE_MIN" ]; then
    say "boot_deadline_s = ${deadline} is below ${DEADLINE_MIN}; using ${DEADLINE_MIN}"
    deadline="$DEADLINE_MIN"
elif [ "$deadline" -gt "$DEADLINE_MAX" ]; then
    say "boot_deadline_s = ${deadline} is above ${DEADLINE_MAX}; using ${DEADLINE_MAX}"
    deadline="$DEADLINE_MAX"
fi

# One property per call. `systemctl show -p A -p B` prints them in systemd's own order rather than
# the order asked for, which makes a multi-line parse quietly wrong the day that order changes.
property() {
    # $1 property, $2 unit
    systemctl show -p "$1" --value "$2" 2>/dev/null || true
}

while :; do
    # No uptime source means no clock to wait on: judge once, as it stands.
    now="$(uptime_secs)"
    [ -n "$now" ] || now="$deadline"

    failed=""
    unsettled=""
    for unit in $MEMBERS; do
        state="$(property ActiveState "$unit")"
        restarts="$(property NRestarts "$unit")"

        # An unloaded unit answers with an empty string for both. Treated as zero rather than special
        # cased: it then fails both conditions below, which is the answer we want anyway.
        case "$restarts" in
            ''|*[!0-9]*) restarts=0 ;;
        esac

        if [ "$state" = failed ] || [ "$restarts" -ge "$MAX_RESTARTS" ]; then
            failed="${failed}${failed:+, }${unit} (${state:-unknown}, ${restarts} restarts)"
            continue
        fi

        # Not loaded: the release predates it, and there is nothing to wait for.
        [ -n "$state" ] && [ "$(property LoadState "$unit")" != not-found ] || continue

        # Microseconds of CLOCK_MONOTONIC, against /proc/uptime's CLOCK_BOOTTIME seconds. The two
        # differ only by time spent suspended, which a robot booting does not do.
        since="$(property ActiveEnterTimestampMonotonic "$unit")"
        case "$since" in
            ''|*[!0-9]*) since=0 ;;
        esac
        if [ "$state" != active ] || [ $((now - since / 1000000)) -lt "$SETTLE" ]; then
            unsettled="${unsettled}${unsettled:+, }${unit} (${state}, ${restarts} restarts)"
        fi
    done

    [ -z "$failed" ] && [ -n "$unsettled" ] && [ "$now" -lt "$deadline" ] || break
    sleep "$POLL"
done

if [ -z "$failed" ]; then
    if [ -z "$unsettled" ]; then
        say "every daemon came up, settled ${now}s after boot; nothing to do"
    else
        say "the ${deadline}s deadline passed with ${unsettled} not settled; none failed, and that
  is left alone; nothing to do"
    fi
    exit 0
fi
say "did not come up: ${failed}"

if [ -n "$DRY_RUN" ]; then
    say "would hand over to robot-rescue because: ${failed}"
    exit 3
fi

# `[update_gate] auto_rollback`, the owner's layer winning. Off unless a layer says exactly `true`:
# anything this cannot read leaves the release where it is, which is the default.
auto_rollback=false
for layer in "$PARAMS_BASE" "$PARAMS_USER"; do
    value="$(gate_key_in "$layer" auto_rollback)"
    [ -n "$value" ] && auto_rollback="$value"
done
if [ "$auto_rollback" != true ]; then
    say "auto_rollback is off ([update_gate] in robotd.toml), so the release stays: ${failed}.
  To fall back to golden by hand: robot-rescue --reboot"
    exit 2
fi

[ -x "$RESCUE" ] || {
    say "no rescue at ${RESCUE}, so there is nothing to hand over to.
  The board is running a release whose daemons did not start: ${failed}"
    exit 1
}

# The rescue owns every remaining decision — whether a golden exists, whether `current` is already
# golden, whether a previous attempt already tried this. It exits 2 when it declines, which is not a
# failure of this check: the question was asked and answered.
say "handing over to robot-rescue: ${failed}"
"$RESCUE" --reboot --because "boot check: ${failed}" || {
    status=$?
    [ "$status" = 2 ] && exit 0
    exit "$status"
}
