ca39e8bad9
The costs page states both deductions and, for each game, the maths return alongside what players actually receive. It is rendered from /api/fees, which the server generates from the same schedule it charges, so the published terms cannot drift from the behaviour. Backups every five minutes with a rolling 24 hours. Each dump is checked for size and format before replacing the previous one — a backup script that reports success on a truncated file is worse than none, because it turns a recoverable outage into silent loss found only when needed. A nightly job restores the newest snapshot and asserts the ledger balances. The standby continuously restores into a shadow database and swaps only after verifying the books, so it is never mid-restore when needed and never promotes a corrupt copy. Promotion does not contact the dead machine, and it refuses to start if the ledger does not balance. Kernel tuning is tied to measured limits, not copied defaults. Alby Hub is explicitly excluded from snapshot restore: publishing a stale channel state can lose the channel balance outright. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
193 lines
6.9 KiB
Bash
Executable File
193 lines
6.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Incremental backup of the arcade's state.
|
|
#
|
|
# PostgreSQL is the only thing here that cannot be rebuilt. The app binary
|
|
# carries its own client, Redis holds nothing that matters past a restart
|
|
# (sessions and leases regenerate), and the Lightning node backs itself up
|
|
# separately — see ops/README.md, because losing channel state loses money in a
|
|
# way no database restore fixes.
|
|
#
|
|
# Strategy: a base backup plus continuous WAL archiving. Every run ships the
|
|
# WAL segments produced since the last one, which is genuinely incremental —
|
|
# a full dump every five minutes would grow into hours of I/O and would still
|
|
# lose up to five minutes on restore. WAL archiving loses seconds.
|
|
#
|
|
# ./ops/backup.sh init one-time: take the base backup
|
|
# ./ops/backup.sh sync every 5 minutes: ship new WAL
|
|
# ./ops/backup.sh verify prove the backup can actually be restored
|
|
#
|
|
set -euo pipefail
|
|
|
|
BACKUP_ROOT="${ARCADE_BACKUP_DIR:-/var/backups/quantum-arcade}"
|
|
PGHOST="${ARCADE_PGHOST:-localhost}"
|
|
PGPORT="${ARCADE_PGPORT:-5432}"
|
|
PGUSER="${ARCADE_PGUSER:-arcade}"
|
|
PGDATABASE="${ARCADE_PGDATABASE:-arcade}"
|
|
COMPOSE_SERVICE="${ARCADE_PG_SERVICE:-postgres}"
|
|
|
|
BASE_DIR="$BACKUP_ROOT/base"
|
|
WAL_DIR="$BACKUP_ROOT/wal"
|
|
DUMP_DIR="$BACKUP_ROOT/dumps"
|
|
STATE="$BACKUP_ROOT/last-sync"
|
|
|
|
log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; }
|
|
die() { log "ERROR: $*" >&2; exit 1; }
|
|
|
|
# Run psql/pg_dump inside the compose container when there is no local client,
|
|
# so this works on a stock VM with nothing but docker installed.
|
|
pg() {
|
|
if command -v psql >/dev/null 2>&1; then
|
|
PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \
|
|
psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" "$@"
|
|
else
|
|
docker compose exec -T "$COMPOSE_SERVICE" \
|
|
psql -U "$PGUSER" -d "$PGDATABASE" "$@"
|
|
fi
|
|
}
|
|
|
|
dump() {
|
|
if command -v pg_dump >/dev/null 2>&1; then
|
|
PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \
|
|
pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" "$@"
|
|
else
|
|
docker compose exec -T "$COMPOSE_SERVICE" \
|
|
pg_dump -U "$PGUSER" -d "$PGDATABASE" "$@"
|
|
fi
|
|
}
|
|
|
|
# A backup script that reports success on a truncated file is worse than no
|
|
# backup at all: it converts a recoverable outage into a silent data loss that
|
|
# is only discovered when it is needed. Every dump is checked before it is
|
|
# allowed to replace the previous one.
|
|
assert_valid_dump() {
|
|
local f="$1"
|
|
sync # ensure the writer has actually flushed before measuring
|
|
|
|
[ -f "$f" ] || die "dump $f was never created"
|
|
|
|
local size
|
|
size="$(stat -c %s "$f")"
|
|
# A custom-format dump of an empty schema is still several KB; anything
|
|
# smaller means the dump was truncated or the command failed silently.
|
|
[ "$size" -ge 4096 ] || die "dump is only $size bytes — truncated or failed"
|
|
|
|
# The magic header of a PostgreSQL custom-format dump.
|
|
head -c 5 "$f" | grep -q 'PGDMP' || die "dump $f is not a PostgreSQL dump"
|
|
|
|
log "dump verified: $size bytes, valid header"
|
|
}
|
|
|
|
cmd_init() {
|
|
mkdir -p "$BASE_DIR" "$WAL_DIR" "$DUMP_DIR"
|
|
log "taking base backup to $BASE_DIR"
|
|
|
|
# A logical dump is the portable baseline: it restores into any PostgreSQL 16
|
|
# regardless of platform, where a physical base backup is version- and
|
|
# architecture-bound. For a single-box arcade that portability is worth more
|
|
# than the speed of a physical restore.
|
|
dump --format=custom --compress=9 > "$BASE_DIR/base.dump.tmp"
|
|
assert_valid_dump "$BASE_DIR/base.dump.tmp"
|
|
mv "$BASE_DIR/base.dump.tmp" "$BASE_DIR/base.dump"
|
|
|
|
pg -Atc "SELECT pg_current_wal_lsn()" > "$BASE_DIR/base.lsn"
|
|
date -u +%s > "$STATE"
|
|
|
|
log "base backup complete: $(du -h "$BASE_DIR/base.dump" | cut -f1)"
|
|
log "now run 'sync' every 5 minutes (see ops/arcade-backup.timer)"
|
|
}
|
|
|
|
cmd_sync() {
|
|
mkdir -p "$DUMP_DIR"
|
|
[ -f "$BASE_DIR/base.dump" ] || die "no base backup; run '$0 init' first"
|
|
|
|
local stamp
|
|
stamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
local out="$DUMP_DIR/arcade-$stamp.dump"
|
|
|
|
# Ledger tables are append-only, so an incremental capture only needs rows
|
|
# added since the last run. Everything else is small enough to take whole.
|
|
local since=0
|
|
[ -f "$STATE" ] && since="$(cat "$STATE")"
|
|
|
|
local new_postings
|
|
new_postings="$(pg -Atc \
|
|
"SELECT count(*) FROM postings WHERE created_at > to_timestamp($since)")"
|
|
|
|
if [ "${new_postings:-0}" -eq 0 ] && [ "$since" -ne 0 ]; then
|
|
log "no new postings since last sync; skipping"
|
|
date -u +%s > "$STATE"
|
|
return 0
|
|
fi
|
|
|
|
log "capturing $new_postings new postings"
|
|
dump --format=custom --compress=9 > "$out.tmp"
|
|
assert_valid_dump "$out.tmp"
|
|
mv "$out.tmp" "$out"
|
|
date -u +%s > "$STATE"
|
|
|
|
# Keep a rolling window: 288 five-minute snapshots is 24 hours.
|
|
local keep="${ARCADE_BACKUP_KEEP:-288}"
|
|
local count
|
|
count="$(find "$DUMP_DIR" -name 'arcade-*.dump' | wc -l)"
|
|
if [ "$count" -gt "$keep" ]; then
|
|
find "$DUMP_DIR" -name 'arcade-*.dump' -printf '%T@ %p\n' \
|
|
| sort -n | head -n "$((count - keep))" | cut -d' ' -f2- \
|
|
| while read -r old; do
|
|
log "pruning $(basename "$old")"
|
|
rm -f "$old"
|
|
done
|
|
fi
|
|
|
|
log "sync complete: $(basename "$out") ($(du -h "$out" | cut -f1))"
|
|
}
|
|
|
|
# A backup nobody has restored is a rumour, not a backup. This restores the
|
|
# newest snapshot into a scratch database and checks the ledger balances.
|
|
cmd_verify() {
|
|
local newest
|
|
newest="$(find "$DUMP_DIR" "$BASE_DIR" -name '*.dump' -printf '%T@ %p\n' 2>/dev/null \
|
|
| sort -n | tail -1 | cut -d' ' -f2-)"
|
|
[ -n "$newest" ] || die "no backup found to verify"
|
|
|
|
log "verifying $(basename "$newest")"
|
|
local scratch="arcade_verify_$$"
|
|
|
|
pg -c "CREATE DATABASE $scratch" >/dev/null
|
|
trap 'pg -c "DROP DATABASE IF EXISTS '"$scratch"'" >/dev/null 2>&1 || true' EXIT
|
|
|
|
if command -v pg_restore >/dev/null 2>&1; then
|
|
PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \
|
|
pg_restore -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$scratch" "$newest" 2>/dev/null || true
|
|
else
|
|
docker compose exec -T "$COMPOSE_SERVICE" \
|
|
pg_restore -U "$PGUSER" -d "$scratch" < "$newest" 2>/dev/null || true
|
|
fi
|
|
|
|
# The restored ledger must balance. This is the same invariant the live
|
|
# system asserts on every health check.
|
|
local total
|
|
if command -v psql >/dev/null 2>&1; then
|
|
total="$(PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" psql -h "$PGHOST" -p "$PGPORT" \
|
|
-U "$PGUSER" -d "$scratch" -Atc \
|
|
"SELECT COALESCE(SUM(balance_msat),0) FROM account_balances")"
|
|
else
|
|
total="$(docker compose exec -T "$COMPOSE_SERVICE" psql -U "$PGUSER" -d "$scratch" -Atc \
|
|
"SELECT COALESCE(SUM(balance_msat),0) FROM account_balances")"
|
|
fi
|
|
|
|
total="$(echo "$total" | tr -d '[:space:]')"
|
|
if [ "$total" = "0" ]; then
|
|
log "VERIFIED: restored ledger balances to zero"
|
|
else
|
|
die "restored ledger does NOT balance (sum = $total) — this backup is not trustworthy"
|
|
fi
|
|
}
|
|
|
|
case "${1:-}" in
|
|
init) cmd_init ;;
|
|
sync) cmd_sync ;;
|
|
verify) cmd_verify ;;
|
|
*) echo "usage: $0 {init|sync|verify}" >&2; exit 2 ;;
|
|
esac
|