Files
openbot/scripts/hosting/openbot-hosted-server
NB 2111ab268a Start a hosted server again when its main process stops responding (#14643)
* Start a hosted server again when its main process stops responding

On 2026-11-21 a hosted server became short of memory. The main process then
stopped in a zygote read and did not respond again. systemd did not start the
service again, because the process did not exit. The server stayed offline
until an operator restarted it.

The launcher now starts a watchdog. Each 30 seconds it reads the run marker,
which main writes each 60 seconds. When the file does not change for 20 reads
(10 minutes), the watchdog stops main with SIGKILL, and systemd starts the
service again. The watchdog compares the file content, not clocks, so a boat
resume does not start it. It does not kill when main did not write the file in
this run, when the disk has less than 1 MB free, or when the pid is not
openbot.

scripts/hosted-watchdog-e2e.ts tests the watchdog with fake apps, and CI runs
it in the reverse-actions-e2e job.

* Keep the watchdog away from the start and the server safe mode

The local review found two cases where the watchdog stopped a process that
did not hang:

- The server safe mode calls recordExit, which stops the run marker
  heartbeat, and then stays alive with no time limit on a hosted server.
  The watchdog killed it every 10 minutes, and the SIGKILL got past
  RestartPreventExitStatus=78.
- Migrations and the database quick_check run on the main thread. On a
  large database they can block the heartbeat for more than 10 minutes.
  The kill rolled back the migration, and the next start tried again.

Now the watchdog counts a read only when the run marker has
"state":"running" and openbot-start-attempts-v1.json has "attempt":null,
which StartAttempts writes after the renderer loads. The watchdog gets the
userData folder in place of the marker path.

--stop and the watchdog now use one helper that checks the pid, the process
name and the cgroup. The E2E test adds the starting and quitting cases, and
starts each fake app only after the first watchdog read, so the watchdog
always sees a change.

* Name the release notes file after the PR branch
2026-11-22 15:08:18 +01:00

166 lines
5.8 KiB
Bash
Executable File

#!/bin/bash
set -euo pipefail
umask 077
service_openbot() {
local comm
[[ $1 =~ ^[1-9][0-9]*$ ]] || return 1
if [ -d /proc/self ]; then
comm=$(cat "/proc/$1/comm" 2>/dev/null) || return 1
else
comm=$(ps -o comm= -p "$1" 2>/dev/null) || return 1
fi
[ "${comm##*/}" = openbot ] && [ "$(cat "/proc/$1/cgroup" 2>/dev/null)" = "$(cat /proc/$$/cgroup 2>/dev/null)" ]
}
if [ "${1:-}" = --stop ]; then
pid=$(cat "$XDG_RUNTIME_DIR/openbot.pid" 2>/dev/null) || pid=""
if service_openbot "$pid" && kill -TERM "$pid" 2>/dev/null; then
while [ -e "/proc/$pid" ]; do sleep 0.2; done
fi
main=${MAINPID:-}
if [[ $main =~ ^[1-9][0-9]*$ ]]; then
for _ in $(seq 1 50); do
grep -qs '^State:[[:space:]]*[^Z[:space:]]' "/proc/$main/status" || break
sleep 0.2
done
fi
exit 0
fi
if [ "${1:-}" = --watchdog ]; then
pid_file=$2 user_data=$3 interval=$4 limit=$5
heartbeat="$user_data/openbot-run-marker-v1.json"
start_attempts="$user_data/openbot-start-attempts-v1.json"
started_and_running() {
grep -qs '"state":"running"' "$heartbeat" && grep -qs '"attempt":null' "$start_attempts"
}
seen=$(cat "$heartbeat" 2>/dev/null) || seen=""
armed=0 misses=0
while sleep "$interval"; do
current=$(cat "$heartbeat" 2>/dev/null) || current=""
if [ "$current" != "$seen" ]; then
seen=$current armed=1 misses=0
continue
fi
[ "$armed" -eq 1 ] || continue
if ! started_and_running; then
misses=0
continue
fi
misses=$((misses + 1))
[ "$misses" -ge "$limit" ] || continue
free_kb=$(df -Pk "$user_data" 2>/dev/null | awk 'NR == 2 { print $4 }') || free_kb=""
if [[ ! $free_kb =~ ^[0-9]+$ ]] || [ "$free_kb" -lt 1024 ]; then continue; fi
pid=$(cat "$pid_file" 2>/dev/null) || pid=""
if ! service_openbot "$pid"; then
armed=0 misses=0
continue
fi
echo "OpenBot did not update its heartbeat for $((interval * limit)) seconds. The watchdog stops it, and systemd starts it again." >&2
kill -KILL "$pid" 2>/dev/null || true
exit 0
done
exit 0
fi
state="$HOME/.config/openbot-hosted"
mode=hosted
if [ -r /opt/OpenBot/hosted/mode ] && [ "$(</opt/OpenBot/hosted/mode)" = self ]; then mode=self; fi
if [ "${1:-}" != --in-session ] && [ "$mode" = self ]; then
mkdir -p "$state"
chmod 0700 "$state"
exec dbus-run-session -- "$0" --in-session
fi
if [ "${1:-}" != --in-session ]; then
for _ in $(seq 1 300); do
[ -s "$state/env" ] && break
sleep 2
done
[ -s "$state/env" ] || { echo "The hosted server env file is missing." >&2; exit 1; }
exec dbus-run-session -- "$0" --in-session
fi
if [ "$mode" = hosted ]; then
while IFS='=' read -r key value; do
case $key in
OPENBOT_HOSTED_HOST_ID | OPENBOT_HOSTED_CLAIM) export "$key=$value" ;;
esac
done <"$state/env"
export OPENBOT_HOSTED_SERVER=1
else
export OPENBOT_SERVER=1
fi
requested_size=""
if [ -r /opt/OpenBot/hosted/openbot.env ]; then
while IFS='=' read -r key value || [ -n "$key" ]; do
case $key in
OPENBOT_AUTH_API_URL) export "$key=$value" ;;
OPENBOT_DISPLAY_SIZE) requested_size=$value ;;
esac
done </opt/OpenBot/hosted/openbot.env
fi
display_size() {
if [[ $1 =~ ^([1-9][0-9]{2,3})x([1-9][0-9]{2,3})$ ]] &&
[ "${BASH_REMATCH[1]}" -ge 640 ] && [ "${BASH_REMATCH[1]}" -le 2576 ] &&
[ "${BASH_REMATCH[2]}" -ge 480 ] && [ "${BASH_REMATCH[2]}" -le 2576 ]; then
echo "$1"
return
fi
[ -z "$1" ] || echo "OPENBOT_DISPLAY_SIZE is not WIDTHxHEIGHT from 640x480 to 2576x2576. The display is 1280x800." >&2
echo 1280x800
}
if [ ! -s "$state/keyring-password" ] && [ -e /srv/openbot-hosted/keyrings/login.keyring ]; then
echo "The keyring exists, but its password file is missing." >&2
exit 1
fi
if [ ! -s "$state/keyring-password" ]; then
temporary=$(mktemp "$state/keyring-password.XXXXXX")
head -c 32 /dev/urandom | base64 | tr -d '\n' >"$temporary"
mv -f "$temporary" "$state/keyring-password"
fi
export XDG_DATA_HOME=/srv/openbot-hosted
keyrings="$XDG_DATA_HOME/keyrings"
mkdir -p "$keyrings"
printf login >"$keyrings/default"
unlocked=$(gnome-keyring-daemon --unlock --components=secrets <"$state/keyring-password")
eval "$unlocked"
started=$(gnome-keyring-daemon --start --components=secrets)
eval "$started"
unset XDG_DATA_HOME
export GNOME_KEYRING_CONTROL
(
for _ in $(seq 1 300); do
case $(findmnt -n -o FSTYPE -T "$keyrings" || true) in fuse*) sleep 2 ;; *) break ;; esac
done
sleep 10
for keyring in "$keyrings"/*.keyring; do
[ -e "$keyring" ] || continue
find "$keyrings" -maxdepth 1 -name "${keyring##*/}.temp-*" ! -samefile "$keyring" -delete
done
) &
"$0" --watchdog "$XDG_RUNTIME_DIR/openbot.pid" "${XDG_CONFIG_HOME:-$HOME/.config}/OpenBot" 30 20 &
openbot=(bash -c 'echo "$$" >"$XDG_RUNTIME_DIR/openbot.pid" && exec "$@"' openbot-hosted-server
/opt/OpenBot/app/openbot --password-store=gnome-libsecret)
if [ "$mode" = hosted ] && [ -e /etc/systemd/system/display-manager.service ]; then
search=0
pgrep -u "$(id -u)" -x xauth >/dev/null 2>&1 || search=$?
if [ "$search" -eq 1 ]; then rm -f "$HOME/.Xauthority-c" "$HOME/.Xauthority-l"; fi
for _ in $(seq 1 60); do
if DISPLAY=:0 XAUTHORITY="$HOME/.Xauthority" xdpyinfo >/dev/null 2>&1; then
export OPENBOT_AGENT_DISPLAY=:0
exec xvfb-run -a -f "$HOME/.Xauthority" -s "-screen 0 1280x800x24 -nolisten tcp" "${openbot[@]}"
fi
sleep 2
done
echo "The boat desktop session did not start. OpenBot uses two virtual displays." >&2
fi
exec xvfb-run -a -s "-screen 0 $(display_size "$requested_size")x24 -nolisten tcp -wr -noreset" \
bash -c 'command -v xsetroot >/dev/null && xsetroot -cursor_name left_ptr; if [ -r "$1" ]; then bash "$1" </dev/null & fi; shift; export OPENBOT_AGENT_DISPLAY=$DISPLAY; exec xvfb-run -a -f "$XAUTHORITY" -s "-screen 0 1280x800x24 -nolisten tcp" "$@"' \
openbot-hosted-server /opt/OpenBot/app/resources/hosting/openbot-desktop "${openbot[@]}"