mirror of
https://github.com/nightly-labs/openbot.git
synced 2026-11-23 02:01:26 +08:00
* Start a hosted server again when its main process stops responding On 2026-11-21 a hosted server became short of memory. The main process then stopped in a zygote read and did not respond again. systemd did not start the service again, because the process did not exit. The server stayed offline until an operator restarted it. The launcher now starts a watchdog. Each 30 seconds it reads the run marker, which main writes each 60 seconds. When the file does not change for 20 reads (10 minutes), the watchdog stops main with SIGKILL, and systemd starts the service again. The watchdog compares the file content, not clocks, so a boat resume does not start it. It does not kill when main did not write the file in this run, when the disk has less than 1 MB free, or when the pid is not openbot. scripts/hosted-watchdog-e2e.ts tests the watchdog with fake apps, and CI runs it in the reverse-actions-e2e job. * Keep the watchdog away from the start and the server safe mode The local review found two cases where the watchdog stopped a process that did not hang: - The server safe mode calls recordExit, which stops the run marker heartbeat, and then stays alive with no time limit on a hosted server. The watchdog killed it every 10 minutes, and the SIGKILL got past RestartPreventExitStatus=78. - Migrations and the database quick_check run on the main thread. On a large database they can block the heartbeat for more than 10 minutes. The kill rolled back the migration, and the next start tried again. Now the watchdog counts a read only when the run marker has "state":"running" and openbot-start-attempts-v1.json has "attempt":null, which StartAttempts writes after the renderer loads. The watchdog gets the userData folder in place of the marker path. --stop and the watchdog now use one helper that checks the pid, the process name and the cgroup. The E2E test adds the starting and quitting cases, and starts each fake app only after the first watchdog read, so the watchdog always sees a change. * Name the release notes file after the PR branch
166 lines
5.8 KiB
Bash
Executable File
166 lines
5.8 KiB
Bash
Executable File
#!/bin/bash
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
service_openbot() {
|
|
local comm
|
|
[[ $1 =~ ^[1-9][0-9]*$ ]] || return 1
|
|
if [ -d /proc/self ]; then
|
|
comm=$(cat "/proc/$1/comm" 2>/dev/null) || return 1
|
|
else
|
|
comm=$(ps -o comm= -p "$1" 2>/dev/null) || return 1
|
|
fi
|
|
[ "${comm##*/}" = openbot ] && [ "$(cat "/proc/$1/cgroup" 2>/dev/null)" = "$(cat /proc/$$/cgroup 2>/dev/null)" ]
|
|
}
|
|
|
|
if [ "${1:-}" = --stop ]; then
|
|
pid=$(cat "$XDG_RUNTIME_DIR/openbot.pid" 2>/dev/null) || pid=""
|
|
if service_openbot "$pid" && kill -TERM "$pid" 2>/dev/null; then
|
|
while [ -e "/proc/$pid" ]; do sleep 0.2; done
|
|
fi
|
|
main=${MAINPID:-}
|
|
if [[ $main =~ ^[1-9][0-9]*$ ]]; then
|
|
for _ in $(seq 1 50); do
|
|
grep -qs '^State:[[:space:]]*[^Z[:space:]]' "/proc/$main/status" || break
|
|
sleep 0.2
|
|
done
|
|
fi
|
|
exit 0
|
|
fi
|
|
if [ "${1:-}" = --watchdog ]; then
|
|
pid_file=$2 user_data=$3 interval=$4 limit=$5
|
|
heartbeat="$user_data/openbot-run-marker-v1.json"
|
|
start_attempts="$user_data/openbot-start-attempts-v1.json"
|
|
started_and_running() {
|
|
grep -qs '"state":"running"' "$heartbeat" && grep -qs '"attempt":null' "$start_attempts"
|
|
}
|
|
seen=$(cat "$heartbeat" 2>/dev/null) || seen=""
|
|
armed=0 misses=0
|
|
while sleep "$interval"; do
|
|
current=$(cat "$heartbeat" 2>/dev/null) || current=""
|
|
if [ "$current" != "$seen" ]; then
|
|
seen=$current armed=1 misses=0
|
|
continue
|
|
fi
|
|
[ "$armed" -eq 1 ] || continue
|
|
if ! started_and_running; then
|
|
misses=0
|
|
continue
|
|
fi
|
|
misses=$((misses + 1))
|
|
[ "$misses" -ge "$limit" ] || continue
|
|
free_kb=$(df -Pk "$user_data" 2>/dev/null | awk 'NR == 2 { print $4 }') || free_kb=""
|
|
if [[ ! $free_kb =~ ^[0-9]+$ ]] || [ "$free_kb" -lt 1024 ]; then continue; fi
|
|
pid=$(cat "$pid_file" 2>/dev/null) || pid=""
|
|
if ! service_openbot "$pid"; then
|
|
armed=0 misses=0
|
|
continue
|
|
fi
|
|
echo "OpenBot did not update its heartbeat for $((interval * limit)) seconds. The watchdog stops it, and systemd starts it again." >&2
|
|
kill -KILL "$pid" 2>/dev/null || true
|
|
exit 0
|
|
done
|
|
exit 0
|
|
fi
|
|
state="$HOME/.config/openbot-hosted"
|
|
mode=hosted
|
|
if [ -r /opt/OpenBot/hosted/mode ] && [ "$(</opt/OpenBot/hosted/mode)" = self ]; then mode=self; fi
|
|
|
|
if [ "${1:-}" != --in-session ] && [ "$mode" = self ]; then
|
|
mkdir -p "$state"
|
|
chmod 0700 "$state"
|
|
exec dbus-run-session -- "$0" --in-session
|
|
fi
|
|
if [ "${1:-}" != --in-session ]; then
|
|
for _ in $(seq 1 300); do
|
|
[ -s "$state/env" ] && break
|
|
sleep 2
|
|
done
|
|
[ -s "$state/env" ] || { echo "The hosted server env file is missing." >&2; exit 1; }
|
|
exec dbus-run-session -- "$0" --in-session
|
|
fi
|
|
|
|
if [ "$mode" = hosted ]; then
|
|
while IFS='=' read -r key value; do
|
|
case $key in
|
|
OPENBOT_HOSTED_HOST_ID | OPENBOT_HOSTED_CLAIM) export "$key=$value" ;;
|
|
esac
|
|
done <"$state/env"
|
|
export OPENBOT_HOSTED_SERVER=1
|
|
else
|
|
export OPENBOT_SERVER=1
|
|
fi
|
|
requested_size=""
|
|
if [ -r /opt/OpenBot/hosted/openbot.env ]; then
|
|
while IFS='=' read -r key value || [ -n "$key" ]; do
|
|
case $key in
|
|
OPENBOT_AUTH_API_URL) export "$key=$value" ;;
|
|
OPENBOT_DISPLAY_SIZE) requested_size=$value ;;
|
|
esac
|
|
done </opt/OpenBot/hosted/openbot.env
|
|
fi
|
|
|
|
display_size() {
|
|
if [[ $1 =~ ^([1-9][0-9]{2,3})x([1-9][0-9]{2,3})$ ]] &&
|
|
[ "${BASH_REMATCH[1]}" -ge 640 ] && [ "${BASH_REMATCH[1]}" -le 2576 ] &&
|
|
[ "${BASH_REMATCH[2]}" -ge 480 ] && [ "${BASH_REMATCH[2]}" -le 2576 ]; then
|
|
echo "$1"
|
|
return
|
|
fi
|
|
[ -z "$1" ] || echo "OPENBOT_DISPLAY_SIZE is not WIDTHxHEIGHT from 640x480 to 2576x2576. The display is 1280x800." >&2
|
|
echo 1280x800
|
|
}
|
|
|
|
if [ ! -s "$state/keyring-password" ] && [ -e /srv/openbot-hosted/keyrings/login.keyring ]; then
|
|
echo "The keyring exists, but its password file is missing." >&2
|
|
exit 1
|
|
fi
|
|
if [ ! -s "$state/keyring-password" ]; then
|
|
temporary=$(mktemp "$state/keyring-password.XXXXXX")
|
|
head -c 32 /dev/urandom | base64 | tr -d '\n' >"$temporary"
|
|
mv -f "$temporary" "$state/keyring-password"
|
|
fi
|
|
export XDG_DATA_HOME=/srv/openbot-hosted
|
|
keyrings="$XDG_DATA_HOME/keyrings"
|
|
mkdir -p "$keyrings"
|
|
printf login >"$keyrings/default"
|
|
unlocked=$(gnome-keyring-daemon --unlock --components=secrets <"$state/keyring-password")
|
|
eval "$unlocked"
|
|
started=$(gnome-keyring-daemon --start --components=secrets)
|
|
eval "$started"
|
|
unset XDG_DATA_HOME
|
|
export GNOME_KEYRING_CONTROL
|
|
|
|
(
|
|
for _ in $(seq 1 300); do
|
|
case $(findmnt -n -o FSTYPE -T "$keyrings" || true) in fuse*) sleep 2 ;; *) break ;; esac
|
|
done
|
|
sleep 10
|
|
for keyring in "$keyrings"/*.keyring; do
|
|
[ -e "$keyring" ] || continue
|
|
find "$keyrings" -maxdepth 1 -name "${keyring##*/}.temp-*" ! -samefile "$keyring" -delete
|
|
done
|
|
) &
|
|
|
|
"$0" --watchdog "$XDG_RUNTIME_DIR/openbot.pid" "${XDG_CONFIG_HOME:-$HOME/.config}/OpenBot" 30 20 &
|
|
|
|
openbot=(bash -c 'echo "$$" >"$XDG_RUNTIME_DIR/openbot.pid" && exec "$@"' openbot-hosted-server
|
|
/opt/OpenBot/app/openbot --password-store=gnome-libsecret)
|
|
if [ "$mode" = hosted ] && [ -e /etc/systemd/system/display-manager.service ]; then
|
|
search=0
|
|
pgrep -u "$(id -u)" -x xauth >/dev/null 2>&1 || search=$?
|
|
if [ "$search" -eq 1 ]; then rm -f "$HOME/.Xauthority-c" "$HOME/.Xauthority-l"; fi
|
|
for _ in $(seq 1 60); do
|
|
if DISPLAY=:0 XAUTHORITY="$HOME/.Xauthority" xdpyinfo >/dev/null 2>&1; then
|
|
export OPENBOT_AGENT_DISPLAY=:0
|
|
exec xvfb-run -a -f "$HOME/.Xauthority" -s "-screen 0 1280x800x24 -nolisten tcp" "${openbot[@]}"
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "The boat desktop session did not start. OpenBot uses two virtual displays." >&2
|
|
fi
|
|
|
|
exec xvfb-run -a -s "-screen 0 $(display_size "$requested_size")x24 -nolisten tcp -wr -noreset" \
|
|
bash -c 'command -v xsetroot >/dev/null && xsetroot -cursor_name left_ptr; if [ -r "$1" ]; then bash "$1" </dev/null & fi; shift; export OPENBOT_AGENT_DISPLAY=$DISPLAY; exec xvfb-run -a -f "$XAUTHORITY" -s "-screen 0 1280x800x24 -nolisten tcp" "$@"' \
|
|
openbot-hosted-server /opt/OpenBot/app/resources/hosting/openbot-desktop "${openbot[@]}"
|