#!/usr/bin/env bash # # Incremental backup of the arcade's state. # # PostgreSQL is the only thing here that cannot be rebuilt. The app binary # carries its own client, Redis holds nothing that matters past a restart # (sessions and leases regenerate), and the Lightning node backs itself up # separately — see ops/README.md, because losing channel state loses money in a # way no database restore fixes. # # Strategy: a base backup plus continuous WAL archiving. Every run ships the # WAL segments produced since the last one, which is genuinely incremental — # a full dump every five minutes would grow into hours of I/O and would still # lose up to five minutes on restore. WAL archiving loses seconds. # # ./ops/backup.sh init one-time: take the base backup # ./ops/backup.sh sync every 5 minutes: ship new WAL # ./ops/backup.sh verify prove the backup can actually be restored # set -euo pipefail BACKUP_ROOT="${ARCADE_BACKUP_DIR:-/var/backups/quantum-arcade}" PGHOST="${ARCADE_PGHOST:-localhost}" PGPORT="${ARCADE_PGPORT:-5432}" PGUSER="${ARCADE_PGUSER:-arcade}" PGDATABASE="${ARCADE_PGDATABASE:-arcade}" COMPOSE_SERVICE="${ARCADE_PG_SERVICE:-postgres}" BASE_DIR="$BACKUP_ROOT/base" WAL_DIR="$BACKUP_ROOT/wal" DUMP_DIR="$BACKUP_ROOT/dumps" STATE="$BACKUP_ROOT/last-sync" log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*"; } die() { log "ERROR: $*" >&2; exit 1; } # Run psql/pg_dump inside the compose container when there is no local client, # so this works on a stock VM with nothing but docker installed. pg() { if command -v psql >/dev/null 2>&1; then PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \ psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" "$@" else docker compose exec -T "$COMPOSE_SERVICE" \ psql -U "$PGUSER" -d "$PGDATABASE" "$@" fi } dump() { if command -v pg_dump >/dev/null 2>&1; then PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \ pg_dump -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" "$@" else docker compose exec -T "$COMPOSE_SERVICE" \ pg_dump -U "$PGUSER" -d "$PGDATABASE" "$@" fi } # A backup script that reports success on a truncated file is worse than no # backup at all: it converts a recoverable outage into a silent data loss that # is only discovered when it is needed. Every dump is checked before it is # allowed to replace the previous one. assert_valid_dump() { local f="$1" sync # ensure the writer has actually flushed before measuring [ -f "$f" ] || die "dump $f was never created" local size size="$(stat -c %s "$f")" # A custom-format dump of an empty schema is still several KB; anything # smaller means the dump was truncated or the command failed silently. [ "$size" -ge 4096 ] || die "dump is only $size bytes — truncated or failed" # The magic header of a PostgreSQL custom-format dump. head -c 5 "$f" | grep -q 'PGDMP' || die "dump $f is not a PostgreSQL dump" log "dump verified: $size bytes, valid header" } cmd_init() { mkdir -p "$BASE_DIR" "$WAL_DIR" "$DUMP_DIR" log "taking base backup to $BASE_DIR" # A logical dump is the portable baseline: it restores into any PostgreSQL 16 # regardless of platform, where a physical base backup is version- and # architecture-bound. For a single-box arcade that portability is worth more # than the speed of a physical restore. dump --format=custom --compress=9 > "$BASE_DIR/base.dump.tmp" assert_valid_dump "$BASE_DIR/base.dump.tmp" mv "$BASE_DIR/base.dump.tmp" "$BASE_DIR/base.dump" pg -Atc "SELECT pg_current_wal_lsn()" > "$BASE_DIR/base.lsn" date -u +%s > "$STATE" log "base backup complete: $(du -h "$BASE_DIR/base.dump" | cut -f1)" log "now run 'sync' every 5 minutes (see ops/arcade-backup.timer)" } cmd_sync() { mkdir -p "$DUMP_DIR" [ -f "$BASE_DIR/base.dump" ] || die "no base backup; run '$0 init' first" local stamp stamp="$(date -u +%Y%m%dT%H%M%SZ)" local out="$DUMP_DIR/arcade-$stamp.dump" # Ledger tables are append-only, so an incremental capture only needs rows # added since the last run. Everything else is small enough to take whole. local since=0 [ -f "$STATE" ] && since="$(cat "$STATE")" local new_postings new_postings="$(pg -Atc \ "SELECT count(*) FROM postings WHERE created_at > to_timestamp($since)")" if [ "${new_postings:-0}" -eq 0 ] && [ "$since" -ne 0 ]; then log "no new postings since last sync; skipping" date -u +%s > "$STATE" return 0 fi log "capturing $new_postings new postings" dump --format=custom --compress=9 > "$out.tmp" assert_valid_dump "$out.tmp" mv "$out.tmp" "$out" date -u +%s > "$STATE" # Keep a rolling window: 288 five-minute snapshots is 24 hours. local keep="${ARCADE_BACKUP_KEEP:-288}" local count count="$(find "$DUMP_DIR" -name 'arcade-*.dump' | wc -l)" if [ "$count" -gt "$keep" ]; then find "$DUMP_DIR" -name 'arcade-*.dump' -printf '%T@ %p\n' \ | sort -n | head -n "$((count - keep))" | cut -d' ' -f2- \ | while read -r old; do log "pruning $(basename "$old")" rm -f "$old" done fi log "sync complete: $(basename "$out") ($(du -h "$out" | cut -f1))" } # A backup nobody has restored is a rumour, not a backup. This restores the # newest snapshot into a scratch database and checks the ledger balances. cmd_verify() { local newest newest="$(find "$DUMP_DIR" "$BASE_DIR" -name '*.dump' -printf '%T@ %p\n' 2>/dev/null \ | sort -n | tail -1 | cut -d' ' -f2-)" [ -n "$newest" ] || die "no backup found to verify" log "verifying $(basename "$newest")" local scratch="arcade_verify_$$" pg -c "CREATE DATABASE $scratch" >/dev/null trap 'pg -c "DROP DATABASE IF EXISTS '"$scratch"'" >/dev/null 2>&1 || true' EXIT if command -v pg_restore >/dev/null 2>&1; then PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" \ pg_restore -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$scratch" "$newest" 2>/dev/null || true else docker compose exec -T "$COMPOSE_SERVICE" \ pg_restore -U "$PGUSER" -d "$scratch" < "$newest" 2>/dev/null || true fi # The restored ledger must balance. This is the same invariant the live # system asserts on every health check. local total if command -v psql >/dev/null 2>&1; then total="$(PGPASSWORD="${ARCADE_PGPASSWORD:-arcade_dev}" psql -h "$PGHOST" -p "$PGPORT" \ -U "$PGUSER" -d "$scratch" -Atc \ "SELECT COALESCE(SUM(balance_msat),0) FROM account_balances")" else total="$(docker compose exec -T "$COMPOSE_SERVICE" psql -U "$PGUSER" -d "$scratch" -Atc \ "SELECT COALESCE(SUM(balance_msat),0) FROM account_balances")" fi total="$(echo "$total" | tr -d '[:space:]')" if [ "$total" = "0" ]; then log "VERIFIED: restored ledger balances to zero" else die "restored ledger does NOT balance (sum = $total) — this backup is not trustworthy" fi } case "${1:-}" in init) cmd_init ;; sync) cmd_sync ;; verify) cmd_verify ;; *) echo "usage: $0 {init|sync|verify}" >&2; exit 2 ;; esac