feat(ops): costs page, 5-minute backups, warm standby, kernel tuning
The costs page states both deductions and, for each game, the maths return alongside what players actually receive. It is rendered from /api/fees, which the server generates from the same schedule it charges, so the published terms cannot drift from the behaviour. Backups every five minutes with a rolling 24 hours. Each dump is checked for size and format before replacing the previous one — a backup script that reports success on a truncated file is worse than none, because it turns a recoverable outage into silent loss found only when needed. A nightly job restores the newest snapshot and asserts the ledger balances. The standby continuously restores into a shadow database and swaps only after verifying the books, so it is never mid-restore when needed and never promotes a corrupt copy. Promotion does not contact the dead machine, and it refuses to start if the ledger does not balance. Kernel tuning is tied to measured limits, not copied defaults. Alby Hub is explicitly excluded from snapshot restore: publishing a stale channel state can lose the channel balance outright. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
drjones
187
ops/standby.sh
Executable file
187
ops/standby.sh
Executable file
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Warm standby: a second machine that can take over when the live one dies.
|
||||
#
|
||||
# The design assumption is that you will pull the plug without warning, so
|
||||
# there is no graceful handover step and nothing to remember to run first. The
|
||||
# standby continuously restores the latest backup and waits. Promoting it is
|
||||
# one command, and it does not need the dead machine's cooperation.
|
||||
#
|
||||
# What this protects: the ledger, accounts, rounds, and fee history.
|
||||
# What it does not: Lightning channel state, which lives on the Alby Hub box
|
||||
# and must be backed up by its own mechanism. Restoring a stale channel state
|
||||
# can lose funds — see ops/README.md before touching it.
|
||||
#
|
||||
# ./ops/standby.sh follow keep restoring the newest backup (run as a service)
|
||||
# ./ops/standby.sh status how far behind the standby is
|
||||
# ./ops/standby.sh promote become live
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
BACKUP_ROOT="${ARCADE_BACKUP_DIR:-/var/backups/quantum-arcade}"
|
||||
DUMP_DIR="$BACKUP_ROOT/dumps"
|
||||
BASE_DIR="$BACKUP_ROOT/base"
|
||||
STATE="$BACKUP_ROOT/standby-restored"
|
||||
PGUSER="${ARCADE_PGUSER:-arcade}"
|
||||
PGDATABASE="${ARCADE_PGDATABASE:-arcade}"
|
||||
COMPOSE_SERVICE="${ARCADE_PG_SERVICE:-postgres}"
|
||||
INTERVAL="${ARCADE_STANDBY_INTERVAL:-300}"
|
||||
|
||||
log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; }
|
||||
die() { log "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
psql_cmd() {
|
||||
if command -v psql >/dev/null 2>&1; then
|
||||
PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \
|
||||
psql -h "${ARCADE_PGHOST:-localhost}" -U "$PGUSER" "$@"
|
||||
else
|
||||
docker compose exec -T "$COMPOSE_SERVICE" psql -U "$PGUSER" "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
restore_cmd() {
|
||||
local db="$1" file="$2"
|
||||
if command -v pg_restore >/dev/null 2>&1; then
|
||||
PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \
|
||||
pg_restore -h "${ARCADE_PGHOST:-localhost}" -U "$PGUSER" \
|
||||
-d "$db" --clean --if-exists "$file" 2>/dev/null || true
|
||||
else
|
||||
docker compose exec -T "$COMPOSE_SERVICE" \
|
||||
pg_restore -U "$PGUSER" -d "$db" --clean --if-exists < "$file" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
newest_backup() {
|
||||
find "$DUMP_DIR" "$BASE_DIR" -name '*.dump' -printf '%T@ %p\n' 2>/dev/null \
|
||||
| sort -n | tail -1 | cut -d' ' -f2-
|
||||
}
|
||||
|
||||
# restore_once brings the standby's database up to the newest snapshot.
|
||||
#
|
||||
# It restores into a shadow database and swaps names on success. Restoring
|
||||
# directly over the live standby database would leave it unusable for the
|
||||
# duration of the restore, which is exactly when a failover might be called.
|
||||
restore_once() {
|
||||
local src
|
||||
src="$(newest_backup)"
|
||||
[ -n "$src" ] || { log "no backup available yet"; return 0; }
|
||||
|
||||
local marker
|
||||
marker="$(stat -c %Y "$src")"
|
||||
if [ -f "$STATE" ] && [ "$(cat "$STATE")" = "$marker" ]; then
|
||||
return 0 # already holding this snapshot
|
||||
fi
|
||||
|
||||
log "restoring $(basename "$src")"
|
||||
local shadow="${PGDATABASE}_shadow"
|
||||
|
||||
psql_cmd -d postgres -c "DROP DATABASE IF EXISTS $shadow" >/dev/null 2>&1 || true
|
||||
psql_cmd -d postgres -c "CREATE DATABASE $shadow" >/dev/null
|
||||
restore_cmd "$shadow" "$src"
|
||||
|
||||
# The restored ledger must balance before it is allowed to become the
|
||||
# standby's live copy. Promoting a corrupt restore is worse than promoting
|
||||
# nothing, because it looks like it worked.
|
||||
local total
|
||||
total="$(psql_cmd -d "$shadow" -Atc \
|
||||
"SELECT COALESCE(SUM(balance_msat),0) FROM account_balances" | tr -d '[:space:]')"
|
||||
if [ "$total" != "0" ]; then
|
||||
psql_cmd -d postgres -c "DROP DATABASE IF EXISTS $shadow" >/dev/null 2>&1 || true
|
||||
die "restored ledger does not balance (sum = $total); standby left on its previous copy"
|
||||
fi
|
||||
|
||||
# Swap: the previous copy becomes the fallback, the new one becomes current.
|
||||
psql_cmd -d postgres -c "DROP DATABASE IF EXISTS ${PGDATABASE}_previous" >/dev/null 2>&1 || true
|
||||
psql_cmd -d postgres -c \
|
||||
"ALTER DATABASE $PGDATABASE RENAME TO ${PGDATABASE}_previous" >/dev/null 2>&1 || true
|
||||
psql_cmd -d postgres -c "ALTER DATABASE $shadow RENAME TO $PGDATABASE" >/dev/null
|
||||
|
||||
echo "$marker" > "$STATE"
|
||||
log "standby now holds $(basename "$src"), ledger verified"
|
||||
}
|
||||
|
||||
cmd_follow() {
|
||||
log "following $DUMP_DIR every ${INTERVAL}s"
|
||||
while true; do
|
||||
restore_once || log "restore failed; keeping previous copy and retrying"
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
local src
|
||||
src="$(newest_backup)"
|
||||
if [ -z "$src" ]; then
|
||||
echo " no backups present"
|
||||
return 1
|
||||
fi
|
||||
local age_backup age_restore now
|
||||
now="$(date -u +%s)"
|
||||
age_backup=$(( now - $(stat -c %Y "$src") ))
|
||||
|
||||
printf ' newest backup %s\n' "$(basename "$src")"
|
||||
printf ' backup age %ds\n' "$age_backup"
|
||||
|
||||
if [ -f "$STATE" ]; then
|
||||
age_restore=$(( now - $(cat "$STATE") ))
|
||||
printf ' standby restored %ds behind live\n' "$age_restore"
|
||||
else
|
||||
printf ' standby restored never\n'
|
||||
fi
|
||||
|
||||
# A standby further behind than two backup intervals is not a standby.
|
||||
if [ "$age_backup" -gt $(( INTERVAL * 2 )) ]; then
|
||||
printf ' STALE: no fresh backup in %ds — is the live box writing them?\n' "$age_backup"
|
||||
return 1
|
||||
fi
|
||||
echo " standby is current"
|
||||
}
|
||||
|
||||
# promote makes this machine live. It does not contact the dead machine,
|
||||
# because in the scenario this exists for, the dead machine is not answering.
|
||||
cmd_promote() {
|
||||
log "promoting this machine to live"
|
||||
|
||||
restore_once || log "could not fetch a newer backup; promoting what is held"
|
||||
|
||||
local total
|
||||
total="$(psql_cmd -d "$PGDATABASE" -Atc \
|
||||
"SELECT COALESCE(SUM(balance_msat),0) FROM account_balances" | tr -d '[:space:]')"
|
||||
[ "$total" = "0" ] || die "ledger does not balance (sum = $total); refusing to promote"
|
||||
|
||||
local players rounds
|
||||
players="$(psql_cmd -d "$PGDATABASE" -Atc \
|
||||
"SELECT count(*) FROM accounts WHERE kind='player'" | tr -d '[:space:]')"
|
||||
rounds="$(psql_cmd -d "$PGDATABASE" -Atc \
|
||||
"SELECT COALESCE(max(id),0) FROM rounds" | tr -d '[:space:]')"
|
||||
|
||||
log "ledger verified: $players players, latest round $rounds"
|
||||
|
||||
docker compose up -d >/dev/null
|
||||
log "arcade started"
|
||||
|
||||
cat <<EOF
|
||||
|
||||
This machine is now live.
|
||||
|
||||
Remaining steps, which cannot be automated safely:
|
||||
|
||||
1. Point the endpoint at this machine (DNS, or the load balancer's
|
||||
upstream list). Until that happens, players still reach the dead one.
|
||||
|
||||
2. Confirm the old machine is genuinely down. Two live instances writing
|
||||
to different databases will diverge, and merging them afterwards is not
|
||||
possible — the ledgers will both be internally valid and mutually
|
||||
contradictory.
|
||||
|
||||
3. Check /api/health returns a zero ledger sum before letting players in.
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
case "${1:-status}" in
|
||||
follow) cmd_follow ;;
|
||||
status) cmd_status ;;
|
||||
promote) cmd_promote ;;
|
||||
*) echo "usage: $0 {follow|status|promote}" >&2; exit 2 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user