casino/ops/standby.sh

#!/usr/bin/env bash
#
# Warm standby: a second machine that can take over when the live one dies.
#
# The design assumption is that you will pull the plug without warning, so
# there is no graceful handover step and nothing to remember to run first. The
# standby continuously restores the latest backup and waits. Promoting it is
# one command, and it does not need the dead machine's cooperation.
#
# What this protects: the ledger, accounts, rounds, and fee history.
# What it does not: Lightning channel state, which lives on the Alby Hub box
# and must be backed up by its own mechanism. Restoring a stale channel state
# can lose funds — see ops/README.md before touching it.
#
#   ./ops/standby.sh follow    keep restoring the newest backup (run as a service)
#   ./ops/standby.sh status    how far behind the standby is
#   ./ops/standby.sh promote   become live
#
set -euo pipefail

BACKUP_ROOT="${ARCADE_BACKUP_DIR:-/var/backups/quantum-arcade}"
DUMP_DIR="$BACKUP_ROOT/dumps"
BASE_DIR="$BACKUP_ROOT/base"
STATE="$BACKUP_ROOT/standby-restored"
PGUSER="${ARCADE_PGUSER:-arcade}"
PGDATABASE="${ARCADE_PGDATABASE:-arcade}"
COMPOSE_SERVICE="${ARCADE_PG_SERVICE:-postgres}"
INTERVAL="${ARCADE_STANDBY_INTERVAL:-300}"

log() { printf '%s  %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; }
die() { log "ERROR: $*" >&2; exit 1; }

psql_cmd() {
  if command -v psql >/dev/null 2>&1; then
    PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \
      psql -h "${ARCADE_PGHOST:-localhost}" -U "$PGUSER" "$@"
  else
    docker compose exec -T "$COMPOSE_SERVICE" psql -U "$PGUSER" "$@"
  fi
}

restore_cmd() {
  local db="$1" file="$2"
  if command -v pg_restore >/dev/null 2>&1; then
    PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \
      pg_restore -h "${ARCADE_PGHOST:-localhost}" -U "$PGUSER" \
      -d "$db" --clean --if-exists "$file" 2>/dev/null || true
  else
    docker compose exec -T "$COMPOSE_SERVICE" \
      pg_restore -U "$PGUSER" -d "$db" --clean --if-exists < "$file" 2>/dev/null || true
  fi
}

newest_backup() {
  find "$DUMP_DIR" "$BASE_DIR" -name '*.dump' -printf '%T@ %p\n' 2>/dev/null \
    | sort -n | tail -1 | cut -d' ' -f2-
}

# restore_once brings the standby's database up to the newest snapshot.
#
# It restores into a shadow database and swaps names on success. Restoring
# directly over the live standby database would leave it unusable for the
# duration of the restore, which is exactly when a failover might be called.
restore_once() {
  local src
  src="$(newest_backup)"
  [ -n "$src" ] || { log "no backup available yet"; return 0; }

  local marker
  marker="$(stat -c %Y "$src")"
  if [ -f "$STATE" ] && [ "$(cat "$STATE")" = "$marker" ]; then
    return 0  # already holding this snapshot
  fi

  log "restoring $(basename "$src")"
  local shadow="${PGDATABASE}_shadow"

  psql_cmd -d postgres -c "DROP DATABASE IF EXISTS $shadow" >/dev/null 2>&1 || true
  psql_cmd -d postgres -c "CREATE DATABASE $shadow" >/dev/null
  restore_cmd "$shadow" "$src"

  # The restored ledger must balance before it is allowed to become the
  # standby's live copy. Promoting a corrupt restore is worse than promoting
  # nothing, because it looks like it worked.
  local total
  total="$(psql_cmd -d "$shadow" -Atc \
    "SELECT COALESCE(SUM(balance_msat),0) FROM account_balances" | tr -d '[:space:]')"
  if [ "$total" != "0" ]; then
    psql_cmd -d postgres -c "DROP DATABASE IF EXISTS $shadow" >/dev/null 2>&1 || true
    die "restored ledger does not balance (sum = $total); standby left on its previous copy"
  fi

  # Swap: the previous copy becomes the fallback, the new one becomes current.
  psql_cmd -d postgres -c "DROP DATABASE IF EXISTS ${PGDATABASE}_previous" >/dev/null 2>&1 || true
  psql_cmd -d postgres -c \
    "ALTER DATABASE $PGDATABASE RENAME TO ${PGDATABASE}_previous" >/dev/null 2>&1 || true
  psql_cmd -d postgres -c "ALTER DATABASE $shadow RENAME TO $PGDATABASE" >/dev/null

  echo "$marker" > "$STATE"
  log "standby now holds $(basename "$src"), ledger verified"
}

cmd_follow() {
  log "following $DUMP_DIR every ${INTERVAL}s"
  while true; do
    restore_once || log "restore failed; keeping previous copy and retrying"
    sleep "$INTERVAL"
  done
}

cmd_status() {
  local src
  src="$(newest_backup)"
  if [ -z "$src" ]; then
    echo "  no backups present"
    return 1
  fi
  local age_backup age_restore now
  now="$(date -u +%s)"
  age_backup=$(( now - $(stat -c %Y "$src") ))

  printf '  newest backup      %s\n' "$(basename "$src")"
  printf '  backup age         %ds\n' "$age_backup"

  if [ -f "$STATE" ]; then
    age_restore=$(( now - $(cat "$STATE") ))
    printf '  standby restored   %ds behind live\n' "$age_restore"
  else
    printf '  standby restored   never\n'
  fi

  # A standby further behind than two backup intervals is not a standby.
  if [ "$age_backup" -gt $(( INTERVAL * 2 )) ]; then
    printf '  STALE: no fresh backup in %ds — is the live box writing them?\n' "$age_backup"
    return 1
  fi
  echo "  standby is current"
}

# promote makes this machine live. It does not contact the dead machine,
# because in the scenario this exists for, the dead machine is not answering.
cmd_promote() {
  log "promoting this machine to live"

  restore_once || log "could not fetch a newer backup; promoting what is held"

  local total
  total="$(psql_cmd -d "$PGDATABASE" -Atc \
    "SELECT COALESCE(SUM(balance_msat),0) FROM account_balances" | tr -d '[:space:]')"
  [ "$total" = "0" ] || die "ledger does not balance (sum = $total); refusing to promote"

  local players rounds
  players="$(psql_cmd -d "$PGDATABASE" -Atc \
    "SELECT count(*) FROM accounts WHERE kind='player'" | tr -d '[:space:]')"
  rounds="$(psql_cmd -d "$PGDATABASE" -Atc \
    "SELECT COALESCE(max(id),0) FROM rounds" | tr -d '[:space:]')"

  log "ledger verified: $players players, latest round $rounds"

  docker compose up -d >/dev/null
  log "arcade started"

  cat <<EOF

  This machine is now live.

  Remaining steps, which cannot be automated safely:

    1. Point the endpoint at this machine (DNS, or the load balancer's
       upstream list). Until that happens, players still reach the dead one.

    2. Confirm the old machine is genuinely down. Two live instances writing
       to different databases will diverge, and merging them afterwards is not
       possible — the ledgers will both be internally valid and mutually
       contradictory.

    3. Check /api/health returns a zero ledger sum before letting players in.

EOF
}

case "${1:-status}" in
  follow)  cmd_follow ;;
  status)  cmd_status ;;
  promote) cmd_promote ;;
  *)       echo "usage: $0 {follow|status|promote}" >&2; exit 2 ;;
esac