#!/bin/bash
# Runs several OpenBot containers on one host. Each instance has its own CPU, memory and disk limit,
# its own data and its own account. See docs/docker.md#several-instances-on-one-host.
#
# An instance is the env file <name>.env next to this script and the folder /srv/openbot/<name>.
# /srv/openbot/<name>/data is an ext4 image on a loop device, so the disk limit is the size of that
# file system. Before the mount, the folder belongs to root, so a container that starts before the
# mount cannot write to /data and stops. It never starts with an empty home folder.
#
# All instances use the Docker network `openbot`, on which they cannot reach each other.
# openbot-firewall.service lets them reach only the internet. `setup` installs both once per host.
#
# The owner can update an instance from the app. OpenBot writes an empty request file to
# /srv/openbot/<name>/update-requests, which the container can write, and
# openbot-instance-update@<name>.path runs `update-request <name>` as root. That pulls the image, and
# for an install makes the container again. Root reports in /srv/openbot/<name>/update-status, which
# the container can only read. The container never gets the Docker socket.
#
# `backup` stops an instance, copies all of /data into one archive in OPENBOT_BACKUP_DIR, and starts
# it again. `verify` unpacks a backup into a folder that no container mounts and checks it. `restore`
# replaces /data with a backup and does not start the instance. Root never opens a database or
# unpacks an archive: agents can change every file in /data, so the user of the data does that.
set -euo pipefail

HERE=$(dirname "$(readlink -f "$0")")
COMPOSE=$HERE/compose.yaml
DATA=/srv/openbot
NETWORK=openbot
SUBNET=172.30.0.0/16
FIREWALL=openbot-firewall.service
LOG_SOCKET=/var/lib/openbot-docker/dev/log
UNIT_DIRECTORY=/etc/systemd/system
UPDATE_UNIT=openbot-instance-update
BACKUP_UNIT=openbot-instance-backup
LOCKS=/run
# LXCFS shows each container its own CPU and memory limits in these files. See lxcfs.yaml.
LXCFS=/var/lib/lxcfs
LXCFS_FILES=(proc/cpuinfo proc/stat proc/meminfo proc/swaps)
LXCFS_DROP_IN=/etc/systemd/system/lxcfs.service.d/openbot.conf
# The files without which a backup cannot open: the database and the keyring with its password.
REQUIRED_FILES=(.config/OpenBot/openbot.db .local/share/keyrings/login.keyring .config/openbot-container/keyring-password)
SHARED_DATABASE=OpenBot/Shared/Data/agent-data.db

fail() {
  echo "$*" >&2
  exit 1
}

usage() {
  cat <<'EOF'
Usage: openbot-instance <command>

  setup                       Install the firewall and the network of the instances. Run it
                              once per host, and again after a change to /etc/openbot-docker/deny.
  create <name> --cpus <n> --mem <n>g --disk <n>G
                              Make an instance and start it. Then sign it in with
                              `openbot-instance run <name> login`.
  list                        Show the instances, their state and disk use.
  up | down | restart <name>  Start, stop or restart an instance. The data stays.
  logs <name> [-f]            Show the log of an instance.
  upgrade <name> | --all      Pull the image and make the container again.
  update-request <name>       Run by openbot-instance-update@<name>.service when the app asks
                              for an update.
  run <name> <command>        Run `openbot <command>` in the instance, for example
                              status, login, mobile-connect or name.
  backup <name> [--verify]    Stop the instance, copy /data to OPENBOT_BACKUP_DIR, and start it
                              again. With --verify, then verify the new backup.
  verify <name> [<backup>]    Unpack a backup, the newest when none is given, into a temporary
                              folder and check its databases and keys.
  restore <name> <backup> [--replace]
                              Replace /data with a backup. The instance stays stopped.
  backups <name> [--json]     Show the backups and the last backup and verification. Exits with
                              1 when the last one failed or stopped, or the newest backup is
                              overdue.

Change the CPU or memory limit in <name>.env, then run `openbot-instance up <name>`. Set
OPENBOT_BACKUP_DIR, OPENBOT_BACKUP_KEEP, OPENBOT_BACKUP_MAX_AGE and OPENBOT_BACKUP_WORK there
for backups.
EOF
}

check_name() {
  [[ $1 =~ ^[a-z0-9][a-z0-9-]{0,23}$ ]] ||
    fail "An instance name has 1 to 24 characters: a-z, 0-9 and -. It starts with a letter or a digit."
}

env_file() {
  echo "$HERE/$1.env"
}

compose() {
  local name=$1
  shift
  check_name "$name"
  [ -f "$(env_file "$name")" ] || fail "There is no instance $name."
  local files=(--file "$COMPOSE")
  # Without LXCFS, an instance still starts, so an update from the app works. OpenBot then limits
  # only the Node programs of agents to the CPU limit (resources/agent-runtime/cpu-limit.cjs).
  if lxcfs_ready; then files+=(--file "$HERE/lxcfs.yaml"); fi
  docker compose --project-name "openbot-$name" "${files[@]}" --env-file "$(env_file "$name")" "$@"
}

# LXCFS runs and has all the files of lxcfs.yaml. Docker cannot start a container whose mount source
# is missing.
lxcfs_ready() {
  systemctl is-active --quiet lxcfs.service || return
  local file
  for file in "${LXCFS_FILES[@]}"; do [ -e "$LXCFS/$file" ] || return; done
}

# Installs LXCFS with --enable-cfs. A restart of LXCFS breaks its files in the containers that run, so
# a change of the options waits until no container uses them. The instances start without LXCFS, so
# each problem here is a warning, and setup runs it after everything that the instances need.
setup_lxcfs() {
  local skip="The instances start without LXCFS and see the CPUs of the host."
  if ! command -v lxcfs >/dev/null; then
    if ! command -v apt-get >/dev/null; then
      echo "Install lxcfs, then run setup again. $skip" >&2
      return 0
    fi
    if ! DEBIAN_FRONTEND=noninteractive apt-get install --yes --quiet lxcfs >/dev/null; then
      echo "lxcfs did not install. $skip" >&2
      return 0
    fi
  fi
  if ! cmp --silent "$HERE/lxcfs-openbot.conf" "$LXCFS_DROP_IN"; then
    install -d -m 0755 "$(dirname "$LXCFS_DROP_IN")"
    install -m 0644 "$HERE/lxcfs-openbot.conf" "$LXCFS_DROP_IN"
    systemctl daemon-reload
    if docker ps --format '{{.Mounts}}' | grep -qF "$LXCFS"; then
      echo "LXCFS has new options, but containers use it. Stop the instances, run 'systemctl restart lxcfs', and start them." >&2
    else
      systemctl restart lxcfs.service || true
    fi
  fi
  if ! systemctl enable --now --quiet lxcfs.service || ! lxcfs_ready; then
    echo "LXCFS does not show its files in $LXCFS. $skip" >&2
    return 0
  fi
  # Each setup says it again until LXCFS runs with the options of the drop-in.
  tr '\0' ' ' <"/proc/$(systemctl show --property MainPID --value lxcfs.service)/cmdline" | grep -q -- --enable-cfs ||
    echo "LXCFS runs without --enable-cfs, so the containers see the CPUs of the host. Stop the instances, run 'systemctl restart lxcfs', and start them." >&2
}

setup() {
  [ "$(id -u)" -eq 0 ] || fail "Run setup as root."
  install -d -m 0755 /etc/openbot-docker
  [ -e /etc/openbot-docker/deny ] ||
    printf '# More addresses that the instances must not reach, one per line, for example servers\n# that trust the address of this host.\n' \
      >/etc/openbot-docker/deny
  sed "s#/opt/openbot-docker/instances/#$HERE/#" "$HERE/$FIREWALL" >"$UNIT_DIRECTORY/$FIREWALL"
  systemctl daemon-reload
  systemctl enable --quiet "$FIREWALL"
  # Docker requires the unit, so a restart of the unit would restart Docker. Apply the rules directly.
  "$HERE/openbot-firewall"
  systemctl start "$FIREWALL"
  docker network inspect "$NETWORK" >/dev/null 2>&1 ||
    docker network create --driver bridge --subnet "$SUBNET" \
      --opt com.docker.network.bridge.name=br-openbot \
      --opt com.docker.network.bridge.enable_icc=false "$NETWORK" >/dev/null
  install -m 0644 "$HERE/openbot-sysctl.conf" /etc/sysctl.d/90-openbot-docker.conf
  sysctl --quiet --load /etc/sysctl.d/90-openbot-docker.conf
  # Container logs: rsyslog writes them to /var/log/openbot, and openbot-logrotate.timer limits them.
  install -d -m 0755 /var/lib/openbot-docker "$(dirname "$LOG_SOCKET")"
  install -d -o syslog -g adm -m 0750 /var/log/openbot
  install -m 0644 "$HERE/openbot-logrotate" /etc/openbot-docker/logrotate.conf
  rm -f /etc/logrotate.d/openbot-docker /etc/systemd/system/docker.service.d/openbot-rsyslog.conf
  install -m 0644 "$HERE/openbot-logrotate.service" "$HERE/openbot-logrotate.timer" /etc/systemd/system/
  install -d -m 0755 /etc/systemd/system/docker.service.d
  install -m 0644 "$HERE/docker-openbot.conf" /etc/systemd/system/docker.service.d/openbot.conf
  # The update requests of the app. `start` enables the path unit of each instance.
  install -m 0644 "$HERE/$UPDATE_UNIT@.path" "$UNIT_DIRECTORY/"
  sed "s#/opt/openbot-docker/instances/#$HERE/#" "$HERE/$UPDATE_UNIT@.service" >"$UNIT_DIRECTORY/$UPDATE_UNIT@.service"
  # Backups. The operator enables the timer of each instance.
  install -m 0644 "$HERE/$BACKUP_UNIT@.timer" "$UNIT_DIRECTORY/"
  sed "s#/opt/openbot-docker/instances/#$HERE/#" "$HERE/$BACKUP_UNIT@.service" >"$UNIT_DIRECTORY/$BACKUP_UNIT@.service"
  systemctl daemon-reload
  systemctl enable --now --quiet openbot-logrotate.timer
  if ! cmp --silent "$HERE/openbot-rsyslog.conf" /etc/rsyslog.d/30-openbot-docker.conf || [ ! -S "$LOG_SOCKET" ]; then
    install -m 0644 "$HERE/openbot-rsyslog.conf" /etc/rsyslog.d/30-openbot-docker.conf
    rsyslogd -N1 >/dev/null || fail "rsyslog refuses /etc/rsyslog.d/30-openbot-docker.conf."
    systemctl restart rsyslog
  fi
  docker info --format '{{.SecurityOptions}}' | grep -q name=userns ||
    echo "Docker runs without userns-remap: UID 1000 in a container is UID 1000 on the host." >&2
  setup_lxcfs
}

# Starts nothing without the firewall, the network and the log socket.
ready() {
  [ -S "$LOG_SOCKET" ] || fail "The log socket $LOG_SOCKET is missing. Run: openbot-instance setup"
  systemctl is-active --quiet "$FIREWALL" || fail "The firewall of the instances is off. Run: openbot-instance setup"
  docker network inspect "$NETWORK" >/dev/null 2>&1 || fail "The network $NETWORK is missing. Run: openbot-instance setup"
  [ -e "$UNIT_DIRECTORY/$UPDATE_UNIT@.service" ] || fail "The update units are missing. Run: openbot-instance setup"
  lxcfs_ready ||
    echo "LXCFS does not run, so the instances see the CPUs and the memory of the host. Run: openbot-instance setup" >&2
}

start() {
  ready
  update_folders "$1" || fail "The update folders of $1 could not be made."
  # update_request runs start in a subshell on the left of `||`, where `set -e` is off: a failed
  # `up` must still be the result of start.
  compose "$1" up --detach || return
  check_swap "$1"
}

# compose.yaml gives no swap above the memory limit. A container that swaps stops for seconds at a
# time when it is full, and its clients lose the connection, so a limit that Docker did not apply is
# reported.
check_swap() {
  local id swap
  id=$(docker inspect --format '{{.Id}}' "openbot-$1") || return 0
  if ! swap=$(cat "/sys/fs/cgroup/system.slice/docker-$id.scope/memory.swap.max" 2>/dev/null); then
    echo "The swap limit of openbot-$1 cannot be read. Check that Docker uses cgroup v2 with the systemd driver and swap accounting." >&2
    return 0
  fi
  [ "$swap" = 0 ] || echo "openbot-$1 can use swap (memory.swap.max is $swap). Check the swap accounting of Docker." >&2
}

# The folders that compose.yaml mounts for updates from the app. Docker would make a missing one as
# root, and the container could not write requests to it. The container can write the request
# folder, so it is a small tmpfs, not a folder on the disk of the host outside the disk limit.
# /etc/fstab mounts it before Docker starts.
update_folders() {
  check_name "$1"
  local folder=$DATA/$1 owner requests
  requests=$DATA/$1/update-requests
  owner=$(data_owner) || return
  install -d -m 0700 "$requests" && install -d -m 0755 "$folder/update-status" || return
  if ! grep -qF " $requests tmpfs " /etc/fstab; then
    printf 'tmpfs %s tmpfs size=64k,nr_inodes=16,mode=0700,uid=%s,gid=%s,nodev,nosuid,noexec,nofail,x-systemd.before=docker.service 0 0\n' \
      "$requests" "${owner%:*}" "${owner#*:}" >>/etc/fstab
    systemctl daemon-reload
  fi
  mountpoint -q "$requests" || mount "$requests" || return
  systemctl enable --now --quiet "$UPDATE_UNIT@$1.path" || return
  # OpenBot offers updates only when this file is there. An instance with a fixed OPENBOT_TAG keeps
  # the version that the administrator chose.
  if grep -q '^OPENBOT_TAG=' "$(env_file "$1")"; then
    rm -f "$folder/update-status/enabled"
  else
    touch "$folder/update-status/enabled"
  fi
}

set_state() {
  printf '%s\n' "$2" >"$1.new"
  mv -f "$1.new" "$1"
}

# The version label of an image or a container. Empty when it is missing or not a release version.
release_version() {
  local version
  version=$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.version"}}' "$1" 2>/dev/null || true)
  if [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "$version"; fi
}

# Pulls the image of instance $1. Writes its version to staged.ready only when it is newer than the
# container: an older OpenBot cannot open a database that a newer one migrated.
stage() {
  local name=$1 status=$DATA/$1/update-status running pulled image
  rm -f "$status/staged.ready"
  running=$(release_version "openbot-$name")
  [ -n "$running" ] || fail "The container openbot-$name has no release version."
  compose "$name" pull --quiet || fail "The image of $name did not download."
  image=$(compose "$name" config --images | head -n 1)
  pulled=$(release_version "$image")
  [ -n "$pulled" ] || fail "The image $image has no release version."
  if [ "$pulled" != "$running" ] && [ "$(printf '%s\n%s\n' "$running" "$pulled" | sort -V | tail -n 1)" = "$pulled" ]; then
    set_state "$status/staged.ready" "$pulled"
  fi
}

# The container owns the request folder, so root only tests a request and removes it. It never reads
# or follows one. An install makes the container again only from a newer image.
update_request() {
  check_name "$1"
  local name=$1 requests=$DATA/$1/update-requests status=$DATA/$1/update-status install=0 result=0 keeper
  exec 9>"$LOCKS/$UPDATE_UNIT-$name.lock"
  flock 9
  if [ -e "$requests/update-install" ] || [ -L "$requests/update-install" ]; then install=1; fi
  # OpenBot waits while the request is there, then reads the state. So the state comes first.
  if [ "$install" -eq 1 ]; then set_state "$status/state" installing; else set_state "$status/state" waiting; fi
  rm -rf -- "$requests/update-stage" "$requests/update-install"
  # OpenBot sees that the request runs while the time of the state changes. `set -e` does not apply
  # in the subshells below, so each step checks its own result.
  # -c: a touch that runs during the end of the request does not make the state again.
  (while sleep 5; do touch -c "$status/state"; done) >/dev/null 2>&1 &
  keeper=$!
  (stage "$name") || result=$?
  if [ "$install" -eq 1 ] && [ -e "$status/staged.ready" ]; then
    rm -f "$status/staged.ready"
    (start "$name") || result=$?
  fi
  kill "$keeper" 2>/dev/null || true
  rm -f "$status/state"
  return "$result"
}

# The host UID:GID of the image user, 1000:1000. With userns-remap, Docker adds the start of the
# ranges of dockremap in /etc/subuid and /etc/subgid.
data_owner() {
  if ! docker info --format '{{.SecurityOptions}}' | grep -q name=userns; then
    echo 1000:1000
    return
  fi
  local uid gid
  uid=$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subuid)
  gid=$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subgid)
  if [ -z "$uid" ] || [ -z "$gid" ]; then
    fail "Docker uses userns-remap, but dockremap has no range in /etc/subuid or /etc/subgid."
  fi
  echo "$((uid + 1000)):$((gid + 1000))"
}

create() {
  [ "$#" -ge 1 ] || fail "Give the name of the instance."
  local name=$1 cpus='' mem='' disk=''
  shift
  check_name "$name"
  while [ "$#" -gt 0 ]; do
    case $1 in
      --cpus) cpus=${2:-} ;;
      --mem) mem=${2:-} ;;
      --disk) disk=${2:-} ;;
      *) fail "Unknown option $1." ;;
    esac
    shift 2 || fail "Give a value for $1."
  done
  [[ $cpus =~ ^[0-9]+(\.[0-9]+)?$ ]] || fail "Give the CPU limit, for example --cpus 8."
  [[ $mem =~ ^[0-9]+[mg]$ ]] || fail "Give the memory limit, for example --mem 32g."
  [[ $disk =~ ^[0-9]+[MGT]$ ]] || fail "Give the disk limit, for example --disk 200G."
  local env folder=$DATA/$name
  env=$(env_file "$name")
  [ ! -e "$env" ] || fail "The instance $name exists."
  [ ! -e "$folder" ] || fail "$folder exists. Move it away or choose another name."
  [ "$(id -u)" -eq 0 ] || fail "Run create as root."
  ready

  install -d -m 0755 "$DATA"
  install -d -m 0700 "$folder"
  # Until the instance exists, a failed step removes what create made, so create can run again.
  trap 'umount "$folder/data" 2>/dev/null || true; rm -rf "$folder"' ERR
  install -d -m 0755 "$folder/data"
  # Reserve all of the space now. A sparse image could grow past the free space of the host, and the
  # database in it would then get write errors.
  fallocate --length "$disk" "$folder/data.img"
  # No blocks kept for root: the user of the image uses all of the limit. A discard would free the
  # reserved space again, so mkfs does none and fstrim.timer skips the mount (X-fstrim.notrim).
  mkfs.ext4 -q -m 0 -E nodiscard "$folder/data.img"
  mount -o loop "$folder/data.img" "$folder/data"
  # The image user. This changes the mounted file system, not the folder under it.
  chown "$(data_owner)" "$folder/data"
  chmod 0700 "$folder/data"
  printf '%s %s ext4 loop,nofail,x-systemd.before=docker.service,X-fstrim.notrim 0 2\n' "$folder/data.img" "$folder/data" >>/etc/fstab
  trap - ERR
  systemctl daemon-reload
  printf '# OPENBOT_DISK is the size of data.img, for information. See docs/docker.md to change it.\nOPENBOT_NAME=%s\nOPENBOT_CPUS=%s\nOPENBOT_MEM=%s\nOPENBOT_DISK=%s\n' \
    "$name" "$cpus" "$mem" "$disk" >"$env"
  start "$name"
  echo "Sign in the instance: openbot-instance run $name login"
}

list() {
  local env name state
  printf '%-24s %-10s %-6s %-6s %s\n' NAME STATE CPUS MEMORY DISK
  for env in "$HERE"/*.env; do
    [ -e "$env" ] || continue
    name=$(basename "$env" .env)
    state=$(docker inspect --format '{{.State.Status}}' "openbot-$name" 2>/dev/null || echo missing)
    # shellcheck source=/dev/null
    (
      source "$env"
      printf '%-24s %-10s %-6s %-6s %s\n' "$name" "$state" "$OPENBOT_CPUS" "$OPENBOT_MEM" \
        "$(df --human-readable --output=used,size "$DATA/$name/data" 2>/dev/null | tail -n 1 | xargs)"
    )
  done
}

# The value of $2 in the env file of instance $1. Empty when it is not set.
setting() {
  sed -n "s/^$2=//p" "$(env_file "$1")" | tail -n 1
}

utc_now() {
  date -u +%Y-%m-%dT%H:%M:%SZ
}

json_string() {
  local value
  value=$(printf '%s' "$1" | tr -d '\000-\010\013\014\016-\037')
  value=${value//\\/\\\\}
  value=${value//\"/\\\"}
  value=${value//$'\t'/\\t}
  value=${value//$'\n'/\\n}
  value=${value//$'\r'/}
  printf '"%s"' "$value"
}

# One flat value of a manifest or status file that this script wrote.
json_value() {
  sed -n "s/^ *\"$2\": \"\{0,1\}\([^\",]*\)\"\{0,1\},\{0,1\}\$/\1/p" "$1" | head -n 1
}

human_bytes() {
  numfmt --to=iec --suffix=B "$1"
}

# The folder of the backups of instance $1. A backup has the keys of the instance, so it is never in
# the data of an instance, where agents could read it.
backup_root() {
  local folder real
  folder=$(setting "$1" OPENBOT_BACKUP_DIR)
  [ -n "$folder" ] || fail "Set OPENBOT_BACKUP_DIR in $(env_file "$1") to the folder of the backups."
  [[ $folder == /* ]] || fail "OPENBOT_BACKUP_DIR must be an absolute path."
  real=$(readlink -m "$folder")
  [[ $real/ != "$(readlink -m "$DATA")"/*/data/* ]] || fail "OPENBOT_BACKUP_DIR must not be in the data of an instance."
  echo "$real/$1"
}

backup_keep() {
  local keep
  keep=$(setting "$1" OPENBOT_BACKUP_KEEP)
  keep=${keep:-7}
  [[ $keep =~ ^[1-9][0-9]{0,3}$ ]] || fail "OPENBOT_BACKUP_KEEP must be a number from 1 to 9999."
  echo "$keep"
}

# The complete backups of instance $1, oldest first. A folder becomes a backup only when it is complete.
complete_backups() {
  local folder
  for folder in "$1"/[0-9]*T[0-9]*Z; do
    [ -f "$folder/manifest.json" ] && echo "$folder"
  done
  return 0
}

# A backup by its folder or its name, or the newest when $2 is empty.
find_backup() {
  local root=$1 wanted=${2:-} found
  if [ -z "$wanted" ]; then
    found=$(complete_backups "$root" | tail -n 1)
    [ -n "$found" ] || fail "There is no complete backup in $root."
  elif [[ $wanted == */* ]]; then
    found=$(readlink -m "$wanted")
  else
    found=$root/$wanted
  fi
  [ -f "$found/manifest.json" ] && [ -f "$found/SHA256SUMS" ] || fail "$found is not a complete backup."
  echo "$found"
}

backup_archive() {
  local archive
  archive=$(json_value "$1/manifest.json" archive)
  [[ $archive =~ ^data\.tar\.(zst|gz)$ ]] && [ -f "$1/$archive" ] || fail "The archive of $1 is missing."
  echo "$archive"
}

decompress() {
  case $1 in
    *.zst) zstd --quiet --decompress --stdout "$2" ;;
    *.gz) gzip --decompress --stdout "$2" ;;
  esac
}

# Runs a command as the user of the data. Root changes to the folder first, so that the user needs no
# access to the folders above it.
# OWNER is the result of data_owner, which each step sets once.
as_owner() {
  setpriv --reuid="${OWNER%:*}" --regid="${OWNER#*:}" --clear-groups --no-new-privs -- "$@"
}

# A container that is not there does not run. When Docker does not answer, nothing is known.
container_running() {
  local state
  if ! state=$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null); then
    docker info >/dev/null 2>&1 || fail "Docker does not answer. Start it, then try again."
    return 1
  fi
  [ "$state" = true ]
}

# A required file must be a file of its own, not a link that root would follow.
check_required_files() {
  local folder=$1 file
  for file in "${REQUIRED_FILES[@]}"; do
    [ -f "$folder/$file" ] && [ ! -L "$folder/$file" ] && [ -s "$folder/$file" ] ||
      fail "$file is missing. The data cannot open without it."
  done
}

# Writes the state of a backup step as one JSON object, and keeps the result of the last success.
record() {
  local status=$1 step=$2 started=$3 result=$4 errors=$5 details=$6 error='' json
  [ "$result" -eq 0 ] || error=$(grep -v '^$' "$errors" | tail -n 1 || true)
  [ "$result" -eq 0 ] || [ -n "$error" ] || error="Stopped with exit code $result."
  json=$(printf '{\n  "step": "%s",\n  "result": "%s",\n  "startedAt": "%s",\n  "finishedAt": "%s",\n  "error": %s%s\n}' \
    "$step" "$([ "$result" -eq 0 ] && echo ok || echo failed)" "$started" "$(utc_now)" \
    "$([ -n "$error" ] && json_string "$error" || echo null)" "$details")
  set_state "$status/$step-attempt.json" "$json"
  [ "$result" -ne 0 ] || set_state "$status/$step-success.json" "$json"
}

# Marks a step as running. A run that a kill or a restart of the host stops keeps this state.
record_start() {
  set_state "$1/$2-attempt.json" "$(printf '{\n  "step": "%s",\n  "result": "running",\n  "startedAt": "%s"\n}' "$2" "$3")"
}

# The result of the last attempt in $1 for instance $2: ok, failed, running, or stopped when the run
# ended without a result. A running step holds the lock of the instance.
attempt_state() {
  local state
  state=$(json_value "$1" result 2>/dev/null || true)
  if [ "$state" = running ] && (flock --nonblock 9) 9>"$LOCKS/$UPDATE_UNIT-$2.lock"; then
    state=stopped
  fi
  echo "$state"
}

# Runs step $2 (backup or verify) as the command in the other arguments, with a details file as its
# last argument, and writes the result to the status folder $1. Exits when the step fails.
run_step() {
  local status=$1 step=$2 started errors details result
  shift 2
  started=$(utc_now)
  record_start "$status" "$step" "$started"
  errors=$(mktemp)
  details=$(mktemp)
  # The errors go to the terminal and to the status. Standard output stays on file descriptor 3.
  set +e
  {
    (
      set -e
      "$@" "$details"
    ) 2>&1 >&3 | tee "$errors" >&2
    result=${PIPESTATUS[0]}
  } 3>&1
  set -e
  record "$status" "$step" "$started" "$result" "$errors" "$(cat "$details")"
  rm -f "$errors" "$details"
  [ "$result" -eq 0 ] || exit "$result"
}

# Holds the lock of instance $1 until the command ends or locks another instance. Updates, backups,
# restores, up, down, restart and upgrade take it, so that none starts or stops the container while
# another runs.
lock_instance() {
  check_name "$1"
  exec 9>"$LOCKS/$UPDATE_UNIT-$1.lock"
  flock 9
}

backup() {
  [ "$#" -ge 1 ] || fail "Give the name of the instance."
  local name=$1 verify_after=0 status made final
  shift
  check_name "$name"
  while [ "$#" -gt 0 ]; do
    case $1 in
      --verify) verify_after=1 ;;
      *) fail "Unknown option $1." ;;
    esac
    shift
  done
  [ "$(id -u)" -eq 0 ] || fail "Run backup as root."
  [ -f "$(env_file "$name")" ] || fail "There is no instance $name."
  status=$DATA/$name/backup-status
  install -d -m 0700 "$status"
  # Updates use the same lock, so a backup never stops a container that an update makes again.
  lock_instance "$name"
  made=$(mktemp)
  # shellcheck disable=SC2064 # The value is fixed now.
  trap "rm -f -- '$made'" EXIT
  run_step "$status" backup backup_steps "$name" "$made"
  final=$(cat "$made")
  # Old backups go only after the new one is complete, and verified when --verify is given. When the
  # verification fails, they all stay.
  [ "$verify_after" -eq 0 ] || verify_locked "$name" "$final"
  prune_backups "$name" "$final"
}

# Removes the partial copy, and starts the instance again when the backup stopped it.
backup_cleanup() {
  rm -rf -- "$PARTIAL"
  [ "$RESTART" -eq 0 ] || docker start "$CONTAINER" >/dev/null || echo "$CONTAINER did not start again." >&2
}

# Runs in a subshell, so its globals end with it. Each failure ends it, and the trap cleans up.
backup_steps() {
  local name=$1 made_file=$2 details_file=$3 data=$DATA/$1/data root used available stamp final archive
  local version image image_id was_running=false exit_code=null stopped_at downtime=0 archive_bytes
  local archive_sha locations compressor=(gzip -1)
  CONTAINER=openbot-$1
  RESTART=0
  OWNER=$(data_owner)
  root=$(backup_root "$name")
  backup_keep "$name" >/dev/null
  mountpoint -q "$data" || fail "$data is not mounted."
  check_required_files "$data"
  install -d -m 0700 "$root"
  # A partial copy is left only by a backup that stopped. The lock is held, so none is running.
  rm -rf -- "$root"/*.partial
  used=$(df --block-size=1 --output=used "$data" | tail -n 1 | tr -d ' ')
  available=$(df --block-size=1 --output=avail "$root" | tail -n 1 | tr -d ' ')
  [ "$available" -gt "$used" ] ||
    fail "The backup can need $(human_bytes "$used") in $root, which has $(human_bytes "$available") free."

  stamp=$(date -u +%Y%m%dT%H%M%SZ)
  # The name is the time to the second. A second backup in the same second takes the next one.
  while [ -e "$root/$stamp" ]; do
    sleep 1
    stamp=$(date -u +%Y%m%dT%H%M%SZ)
  done
  PARTIAL=$root/$stamp.partial
  final=$root/$stamp
  trap backup_cleanup EXIT
  install -d -m 0700 "$PARTIAL"
  archive=data.tar.gz
  if command -v zstd >/dev/null; then
    archive=data.tar.zst
    compressor=(zstd --quiet --threads=0 -3)
  fi
  version=$(release_version "$CONTAINER")
  image=$(docker inspect --format '{{.Config.Image}}' "$CONTAINER" 2>/dev/null || true)
  image_id=$(docker inspect --format '{{.Image}}' "$CONTAINER" 2>/dev/null || true)
  # `down` removes the container. Then the version comes from the image of the instance, so that a
  # restore can refuse an older image.
  if [ -z "$image" ]; then
    image=$(compose "$name" config --images | head -n 1 || true)
    image_id=$(docker image inspect --format '{{.Id}}' "$image" 2>/dev/null || true)
    version=$(release_version "$image")
  fi
  container_running "$CONTAINER" && was_running=true
  if $was_running; then
    RESTART=1
    echo "Stopping $CONTAINER."
    stopped_at=$(date +%s)
    docker stop "$CONTAINER" >/dev/null
    ! container_running "$CONTAINER" || fail "$CONTAINER did not stop."
    exit_code=$(docker inspect --format '{{.State.ExitCode}}' "$CONTAINER")
    [ "$exit_code" = 0 ] ||
      echo "OpenBot stopped with exit code $exit_code. SQLite keeps the last complete transaction." >&2
  fi

  echo "Copying $data."
  locations=$(cd "$data" && find . -mindepth 1 -maxdepth 1 ! -name lost+found -printf '%P\0' |
    sort -z | while IFS= read -r -d '' entry; do json_string "$entry"; printf ', '; done)
  # The user of the data reads the files, so that a link in /data cannot show root's files.
  (cd "$data" && as_owner sha256sum -- "${REQUIRED_FILES[@]}" &&
    if [ -f "$SHARED_DATABASE" ] && [ ! -L "$SHARED_DATABASE" ]; then as_owner sha256sum -- "$SHARED_DATABASE"; fi) \
    >"$PARTIAL/required.sha256"
  tar --create --file=- --directory="$data" --numeric-owner --sparse --one-file-system \
    --exclude=./lost+found . | "${compressor[@]}" >"$PARTIAL/$archive"
  if $was_running; then
    RESTART=0
    docker start "$CONTAINER" >/dev/null
    downtime=$(($(date +%s) - stopped_at))
    echo "Started $CONTAINER again after $downtime s."
  fi

  # Read the archive again from the destination, so that the checksum is of what was written.
  archive_sha=$(sha256sum "$PARTIAL/$archive" | cut -d ' ' -f 1)
  archive_bytes=$(stat --format=%s "$PARTIAL/$archive")
  cp "$(env_file "$name")" "$PARTIAL/instance.env"
  printf '{\n  "format": "openbot-instance-backup",\n  "formatVersion": 1,\n  "instance": "%s",\n  "createdAt": "%s",\n  "openbotVersion": "%s",\n  "image": %s,\n  "imageId": %s,\n  "archive": "%s",\n  "archiveBytes": %s,\n  "archiveSha256": "%s",\n  "dataBytes": %s,\n  "locations": [%s],\n  "excluded": ["lost+found"],\n  "wasRunning": %s,\n  "stopExitCode": %s,\n  "downtimeSeconds": %s\n}\n' \
    "$name" "$(utc_now)" "$version" "$(json_string "$image")" "$(json_string "$image_id")" "$archive" \
    "$archive_bytes" "$archive_sha" "$used" "${locations%, }" "$was_running" "$exit_code" "$downtime" \
    >"$PARTIAL/manifest.json"
  (cd "$PARTIAL" && sha256sum -- "$archive" required.sha256 instance.env) >"$PARTIAL/SHA256SUMS"
  sync --file-system "$PARTIAL/SHA256SUMS"
  mv -T -- "$PARTIAL" "$final"
  trap - EXIT
  sync --file-system "$final"
  echo "$final" >"$made_file"
  printf ',\n  "backup": %s,\n  "archiveBytes": %s,\n  "downtimeSeconds": %s' \
    "$(json_string "$final")" "$archive_bytes" "$downtime" >"$details_file"
  echo "Backup $final: $(human_bytes "$archive_bytes"), $downtime s offline."
}

# Deletes the oldest complete backups of instance $1 above OPENBOT_BACKUP_KEEP. The new backup $2 stays
# also when the clock went back and its name sorts before the others.
prune_backups() {
  local keep existing
  keep=$(backup_keep "$1")
  mapfile -t existing < <(complete_backups "$(backup_root "$1")" | grep -vxF -- "$2")
  if [ "${#existing[@]}" -ge "$keep" ]; then
    rm -rf -- "${existing[@]:0:${#existing[@]}-keep+1}"
  fi
}

verify() {
  [ "$#" -ge 1 ] || fail "Give the name of the instance."
  check_name "$1"
  [ "$(id -u)" -eq 0 ] || fail "Run verify as root."
  [ -f "$(env_file "$1")" ] || fail "There is no instance $1."
  [ "$#" -le 2 ] || fail "Unknown option $3."
  install -d -m 0700 "$DATA/$1/backup-status"
  lock_instance "$1"
  verify_locked "$1" "${2:-}"
}

verify_locked() {
  run_step "$DATA/$1/backup-status" verify verify_steps "$1" "$2"
}

# The number of rows of $2 in database $1, or null when the table is not there. Runs as the user of
# the data in the unpacked folder: -safe refuses ATTACH, extensions and file access.
row_count() {
  local exists
  exists=$(sqlite "$1" "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = '$2';")
  if [ "$exists" = 1 ]; then sqlite "$1" "SELECT count(*) FROM \"$2\";"; else echo null; fi
}

sqlite() {
  as_owner sqlite3 -init /dev/null -batch -noheader -safe "$1" "$2"
}

verify_steps() {
  local name=$1 details_file=$3 root backup archive parent work needed available integrity version
  local warnings=() agents threads messages memories shared_tables=null missing_owners=0 owners warning
  local warning_json=''
  OWNER=$(data_owner)
  command -v sqlite3 >/dev/null || fail "Install sqlite3 3.37 or later to verify a backup."
  root=$(backup_root "$name")
  backup=$(find_backup "$root" "$2")
  archive=$(backup_archive "$backup")
  echo "Verifying $backup."
  (cd "$backup" && sha256sum --check --quiet --strict SHA256SUMS) || fail "The files of $backup do not match their checksums."
  needed=$(json_value "$backup/manifest.json" dataBytes)
  [[ $needed =~ ^[0-9]+$ ]] || fail "The manifest of $backup has no size."
  # No container mounts the temporary folder, and nothing in it runs. SQLite opens a database by its
  # full path, so the user of the data must reach the folder, which the backups do not permit.
  parent=$(setting "$name" OPENBOT_BACKUP_WORK)
  parent=${parent:-/var/tmp}
  [ -d "$parent" ] || fail "OPENBOT_BACKUP_WORK $parent is not a folder."
  work=$(mktemp -d "$parent/openbot-verify.XXXXXX")
  # shellcheck disable=SC2064 # The value is fixed now.
  trap "rm -rf -- '$work'" EXIT
  available=$(df --block-size=1 --output=avail "$work" | tail -n 1 | tr -d ' ')
  [ "$available" -gt "$needed" ] ||
    fail "Verification needs $(human_bytes "$needed") in $parent, which has $(human_bytes "$available") free. Set OPENBOT_BACKUP_WORK."
  chown "$OWNER" "$work"
  as_owner test -w "$work" ||
    fail "The user of the data cannot reach $parent. Set OPENBOT_BACKUP_WORK to a folder that it can reach."
  cd "$work"
  decompress "$archive" "$backup/$archive" | as_owner tar --extract --file=- --preserve-permissions ||
    fail "The archive of $backup does not unpack."
  check_required_files "$work"
  as_owner sha256sum --check --quiet --strict - <"$backup/required.sha256" ||
    fail "The databases or keys in the archive do not match the files that were copied."

  integrity=$(sqlite "${REQUIRED_FILES[0]}" "PRAGMA integrity_check;") || fail "OpenBot cannot open openbot.db."
  [ "$integrity" = ok ] || fail "openbot.db is damaged: $(head -n 1 <<<"$integrity")"
  [ -z "$(sqlite "${REQUIRED_FILES[0]}" "PRAGMA foreign_key_check;")" ] ||
    warnings+=("openbot.db has rows whose references are missing.")
  version=$(sqlite "${REQUIRED_FILES[0]}" "SELECT max(version) FROM schema_migrations;") ||
    fail "openbot.db has no schema version."
  # Agents can change the database, so only a number goes into the JSON.
  [[ $version =~ ^[0-9]+$ ]] || fail "openbot.db has no schema version."
  agents=$(row_count "${REQUIRED_FILES[0]}" projection_agents)
  threads=$(row_count "${REQUIRED_FILES[0]}" projection_threads)
  messages=$(row_count "${REQUIRED_FILES[0]}" projection_thread_messages)
  memories=$(row_count "${REQUIRED_FILES[0]}" projection_agent_memories)

  if [ -f "$SHARED_DATABASE" ]; then
    integrity=$(sqlite "$SHARED_DATABASE" "PRAGMA integrity_check;") || fail "The shared tables cannot open."
    [ "$integrity" = ok ] || fail "The shared tables are damaged: $(head -n 1 <<<"$integrity")"
    shared_tables=$(sqlite "$SHARED_DATABASE" \
      "SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite~_%' ESCAPE '~' AND name <> 'openbot_metadata';")
    # The owner of each shared table is an agent ID, which a restore keeps.
    if [ "$(row_count "$SHARED_DATABASE" openbot_metadata)" != null ] && [ "$agents" != null ]; then
      owners=$(comm -23 <(sqlite "$SHARED_DATABASE" "SELECT DISTINCT owner_agent_id FROM openbot_metadata;" | sort -u) \
        <(sqlite "${REQUIRED_FILES[0]}" "SELECT agent_id FROM projection_agents;" | sort -u))
      [ -z "$owners" ] || missing_owners=$(wc -l <<<"$owners")
      [ "$missing_owners" -eq 0 ] || warnings+=("$missing_owners shared table owners are not agents of this instance.")
    fi
  fi

  for warning in "${warnings[@]}"; do
    echo "Warning: $warning" >&2
    warning_json+="$(json_string "$warning"), "
  done
  printf ',\n  "backup": %s,\n  "schemaVersion": %s,\n  "agents": %s,\n  "threads": %s,\n  "messages": %s,\n  "memories": %s,\n  "sharedTables": %s,\n  "warnings": [%s]' \
    "$(json_string "$backup")" "$version" "$agents" "$threads" "$messages" "$memories" "$shared_tables" \
    "${warning_json%, }" >"$details_file"
  printf '{\n  "verifiedAt": "%s"%s\n}\n' "$(utc_now)" "$(cat "$details_file")" >"$backup/verified.json.new"
  mv -f "$backup/verified.json.new" "$backup/verified.json"
  echo "Verified $backup: schema $version, $agents agents, $threads threads, $messages messages, $memories memories."
}

restore() {
  [ "$#" -ge 2 ] || fail "Give the name of the instance and the backup."
  local name=$1 wanted=$2 replace=0 root backup archive data version target result
  shift 2
  check_name "$name"
  while [ "$#" -gt 0 ]; do
    case $1 in
      --replace) replace=1 ;;
      *) fail "Unknown option $1." ;;
    esac
    shift
  done
  [ "$(id -u)" -eq 0 ] || fail "Run restore as root."
  [ -f "$(env_file "$name")" ] || fail "There is no instance $name."
  data=$DATA/$name/data
  mountpoint -q "$data" || fail "$data is not mounted."
  root=$(backup_root "$name")
  backup=$(find_backup "$root" "$wanted")
  archive=$(backup_archive "$backup")
  (cd "$backup" && sha256sum --check --quiet --strict SHA256SUMS) || fail "The files of $backup do not match their checksums."
  # An older OpenBot cannot open a database that a newer one migrated.
  version=$(json_value "$backup/manifest.json" openbotVersion)
  target=$(release_version "$(compose "$name" config --images | head -n 1)")
  if [ -n "$version" ] && [ -n "$target" ] && [ "$version" != "$target" ] &&
    [ "$(printf '%s\n%s\n' "$version" "$target" | sort -V | tail -n 1)" = "$version" ]; then
    fail "The backup is from OpenBot $version, and the image of $name is $target. Pull a newer image first."
  fi
  [ -n "$target" ] || echo "The version of the image of $name is not known. Pull it with docker compose first." >&2
  [ -n "$version" ] ||
    echo "The backup has no OpenBot version. Make sure that the image of $name is not older than the backup." >&2
  if [ -n "$(find "$data" -mindepth 1 -maxdepth 1 ! -name lost+found -print -quit)" ] && [ "$replace" -eq 0 ]; then
    fail "$data has data. Back it up, then add --replace to delete it and restore $backup."
  fi
  if [ -n "$(find "$data" -mindepth 1 -maxdepth 1 -name '.openbot-replaced.*' -print -quit)" ]; then
    fail "$data has a .openbot-replaced folder from a restore that did not end. Move its contents back or delete it."
  fi
  lock_instance "$name"
  install -d -m 0700 "$DATA/$name/backup-status"
  # A checksum-consistent archive can contain a damaged database. Verify it while the same lock
  # protects the target, before restore_steps stops the container or replaces any data.
  verify_locked "$name" "$backup"
  set +e
  (
    set -e
    restore_steps "$name" "$backup" "$archive"
  )
  result=$?
  set -e
  return "$result"
}

restore_steps() {
  local name=$1 backup=$2 archive=$3 data=$DATA/$1/data container=openbot-$1 needed available entry
  local stage replaced start=:
  OWNER=$(data_owner)
  needed=$(json_value "$backup/manifest.json" dataBytes)
  [[ $needed =~ ^[0-9]+$ ]] || fail "The manifest of $backup has no size."
  if container_running "$container"; then
    start="docker start '$container' >/dev/null"
    # shellcheck disable=SC2064 # The value is fixed now.
    trap "$start" EXIT
    echo "Stopping $container."
    docker stop "$container" >/dev/null
  fi
  ! container_running "$container" || fail "$container did not stop."
  available=$(df --block-size=1 --output=avail "$data" | tail -n 1 | tr -d ' ')
  [ "$available" -gt "$needed" ] ||
    fail "The restore needs $(human_bytes "$needed") free in $data, which has $(human_bytes "$available"). The old data stays until the backup is unpacked. Copy the old data to another disk, delete it from $data, then restore again."
  # mktemp makes a new folder with a new name. It does not follow a link that an agent made in /data
  # before the container stopped.
  stage=$(mktemp -d "$data/.openbot-restore.XXXXXX")
  # shellcheck disable=SC2064 # The value is fixed now.
  trap "rm -rf -- '$stage'; $start" EXIT
  chown "$OWNER" "$stage"
  echo "Unpacking $backup."
  (cd "$stage" && decompress "$archive" "$backup/$archive" |
    as_owner tar --extract --file=- --preserve-permissions) || fail "The archive of $backup does not unpack."
  check_required_files "$stage"
  # From here the old data moves aside. When a move fails, both folders stay for the operator.
  replaced=$(mktemp -d "$data/.openbot-replaced.XXXXXX")
  # shellcheck disable=SC2064 # The value is fixed now.
  trap "echo 'The restore stopped while it moved files. The old data is in $replaced, the backup in $stage.' >&2" EXIT
  cd "$data"
  find . -mindepth 1 -maxdepth 1 ! -name lost+found ! -name "${stage##*/}" ! -name "${replaced##*/}" -print0 |
    while IFS= read -r -d '' entry; do mv -- "$entry" "$replaced/"; done
  find "$stage" -mindepth 1 -maxdepth 1 -print0 |
    while IFS= read -r -d '' entry; do mv -- "$entry" .; done
  rmdir "$stage"
  trap - EXIT
  rm -rf -- "$replaced"
  echo "Restored $backup to $name. It stays stopped."
  echo "Make sure that the instance whose backup this is does not run anywhere else. Then: openbot-instance up $name"
}

backups() {
  [ "$#" -ge 1 ] || fail "Give the name of the instance."
  local name=$1 json=0 status root max_age backup_state verify_state finished overdue=false failed=0 folder list='' file key
  local state
  check_name "$name"
  [ "${2:-}" != --json ] || json=1
  [ -f "$(env_file "$name")" ] || fail "There is no instance $name."
  status=$DATA/$name/backup-status
  root=$(backup_root "$name")
  max_age=$(setting "$name" OPENBOT_BACKUP_MAX_AGE)
  [ -z "$max_age" ] || [[ $max_age =~ ^[1-9][0-9]*$ ]] || fail "OPENBOT_BACKUP_MAX_AGE must be a number of hours."
  backup_state=$(attempt_state "$status/backup-attempt.json" "$name")
  verify_state=$(attempt_state "$status/verify-attempt.json" "$name")
  [ "$backup_state" != failed ] && [ "$backup_state" != stopped ] || failed=1
  [ "$verify_state" != failed ] && [ "$verify_state" != stopped ] || failed=1
  if [ -n "$max_age" ]; then
    finished=$(json_value "$status/backup-success.json" finishedAt 2>/dev/null || true)
    if [ -z "$finished" ] || [ $(($(date +%s) - $(date -d "$finished" +%s))) -gt $((max_age * 3600)) ]; then
      overdue=true
      failed=1
    fi
  fi
  if [ "$json" -eq 1 ]; then
    while IFS= read -r folder; do
      list+="$(json_string "$(basename "$folder")"), "
    done < <(complete_backups "$root")
    printf '{\n"instance": "%s",\n"backups": [%s],\n"overdue": %s,\n"lastBackup": %s,\n"lastVerify": %s,\n' \
      "$name" "${list%, }" "$overdue" "$(json_string "${backup_state:-none}")" "$(json_string "${verify_state:-none}")"
    for key in backup-attempt backup-success verify-attempt verify-success; do
      file=$status/$key.json
      printf '"%s": %s%s\n' "$key" "$([ -f "$file" ] && cat "$file" || echo null)" "$([ "$key" = verify-success ] || echo ,)"
    done
    echo '}'
  else
    printf '%-18s %-10s %s\n' BACKUP SIZE VERIFIED
    while IFS= read -r folder; do
      printf '%-18s %-10s %s\n' "$(basename "$folder")" \
        "$(human_bytes "$(json_value "$folder/manifest.json" archiveBytes)")" \
        "$(json_value "$folder/verified.json" verifiedAt 2>/dev/null || true)"
    done < <(complete_backups "$root")
    for key in backup verify; do
      file=$status/$key-attempt.json
      state=$backup_state
      [ "$key" = backup ] || state=$verify_state
      case $state in
        '') echo "Last $key: none" ;;
        running) echo "Last $key: running since $(json_value "$file" startedAt)" ;;
        stopped) echo "Last $key: stopped without a result. It started at $(json_value "$file" startedAt)." ;;
        ok) echo "Last $key: ok at $(json_value "$file" finishedAt)" ;;
        *) echo "Last $key: failed at $(json_value "$file" finishedAt): $(sed -n 's/^  "error": "\(.*\)",\{0,1\}$/\1/p' "$file")" ;;
      esac
    done
    [ "$overdue" = false ] || echo "The newest backup is older than $max_age hours."
  fi
  return "$failed"
}

[ "${BASH_SOURCE[0]}" = "$0" ] || return 0

command=${1:-}
shift || true
case $command in
  setup) setup ;;
  create) create "$@" ;;
  list) list ;;
  up) lock_instance "${1:-}"; start "$1" ;;
  down) lock_instance "${1:-}"; compose "$1" down ;;
  restart) lock_instance "${1:-}"; compose "$1" restart ;;
  logs) compose "${1:-}" logs "${@:2}" ;;
  upgrade)
    if [ "${1:-}" = --all ]; then
      for env in "$HERE"/*.env; do
        [ -e "$env" ] || continue
        name=$(basename "$env" .env)
        lock_instance "$name"
        compose "$name" pull
        start "$name"
      done
    else
      lock_instance "${1:-}"
      compose "$1" pull
      start "$1"
    fi
    ;;
  update-request) update_request "${1:-}" ;;
  backup) backup "$@" ;;
  verify) verify "$@" ;;
  restore) restore "$@" ;;
  backups) backups "$@" ;;
  run)
    check_name "${1:-}"
    # login asks for the code in a terminal. Scripts and ssh without -t have none.
    terminal=()
    [ ! -t 0 ] || terminal=(--tty)
    docker exec --interactive "${terminal[@]}" "openbot-$1" openbot "${@:2}"
    ;;
  *)
    usage
    [ -z "$command" ] || exit 1
    ;;
esac
