feat(cluster): zero-config horizontal scaling by cloning

An instance decides what it is at startup instead of being told: it
generates its own identity, registers a heartbeat, and campaigns for
each game. Exactly one instance drives a game's rounds and publishes
frames; the rest relay them and forward mutations to the leader. Clone
the VM, boot it, done.

Sessions and the scratch nonce move to Redis. Both were per-instance
state that would have broken behind a load balancer: a token minted by
one clone was unknown to the others, and two clones would have handed
the same nonce to different players, which for the same key means the
same outcome.

Fixes a bug found by running two instances: /api/games read the local
room object, so a follower reported a permanently settled game and its
clients never saw a betting window. Hubs now serve the last frame they
saw, produced or relayed.

Failover measured at 6s after kill -9 on an instance leading two games.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

cmd/arcade/main.go

@@ -14,11 +14,12 @@ import (
 	"os"
 	"os/signal"
 	"strconv"
-	"sync"
+	"strings"
 	"syscall"
 	"time"
 
 	"github.com/coder/websocket"
+	"github.com/drjones/quantum-arcade/pkg/cluster"
 	"github.com/drjones/quantum-arcade/pkg/fair"
 	"github.com/drjones/quantum-arcade/pkg/fixed"
 	"github.com/drjones/quantum-arcade/pkg/identity"
@@ -26,6 +27,7 @@ import (
 	"github.com/drjones/quantum-arcade/pkg/room"
 	"github.com/drjones/quantum-arcade/pkg/scratch"
 	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/redis/go-redis/v9"
 )
 
 //go:embed static
@@ -44,16 +46,21 @@ type server struct {
 	ledger *ledger.Ledger
 	auth   *identity.Authenticator
 	rooms  map[string]*room.Room
+	hubs   map[string]*gameHub
 
-	// sessions maps a bearer token to a verified public key. Sessions live in
-	// memory only: restarting the server signs everyone out, which is fine for
-	// a machine you own and means there is no session store to leak.
-	sessMu   sync.RWMutex
-	sessions map[string]string
+	// Sessions live in Redis rather than instance memory. With several cloned
+	// instances behind one endpoint, a token issued by one must be accepted by
+	// all of them — otherwise every request would have to return to the
+	// instance that happened to handle the sign-in.
+	rdb  *redis.Client
+	node *cluster.Node
 
 	// scratchNonce advances per play so each ticket has a distinct seed.
-	nonceMu      sync.Mutex
-	scratchNonce uint64
+	//
+	// It is drawn from Redis rather than a local counter: with several
+	// instances serving, two clones would otherwise hand the same nonce to
+	// different players, and identical nonces mean identical outcomes for the
+	// same key. The counter is shared, so every ticket is distinct fleet-wide.
 }
 
 func main() {
@@ -78,6 +85,19 @@ func main() {
 		log.Fatalf("database unreachable: %v", err)
 	}
 
+	// Redis carries sessions and cluster coordination. Every cloned instance
+	// points at the same one; that plus the same database is the entire
+	// configuration a clone needs.
+	redisAddr := os.Getenv("ARCADE_REDIS")
+	if redisAddr == "" {
+		redisAddr = "localhost:6379"
+	}
+	rdb := redis.NewClient(&redis.Options{Addr: redisAddr})
+	defer rdb.Close()
+	if err := rdb.Ping(ctx).Err(); err != nil {
+		log.Fatalf("redis unreachable at %s: %v", redisAddr, err)
+	}
+
 	// Every ticket in the catalog must have coherent odds before we serve it.
 	for _, t := range scratch.Catalog {
 		if err := t.Validate(); err != nil {
@@ -86,21 +106,32 @@ func main() {
 	}
 
 	s := &server{
-		pool:     pool,
-		ledger:   ledger.New(pool),
-		auth:     identity.NewAuthenticator(),
-		rooms:    make(map[string]*room.Room),
-		sessions: make(map[string]string),
+		pool:   pool,
+		ledger: ledger.New(pool),
+		auth:   identity.NewAuthenticator(),
+		rooms:  make(map[string]*room.Room),
+		hubs:   make(map[string]*gameHub),
+		rdb:    rdb,
 	}
 
+	// Identity is generated, not configured: a cloned VM boots with its own
+	// id and joins the cluster without anyone editing a file.
+	s.node = cluster.NewNode(rdb, advertiseAddr())
+	if err := s.node.Start(ctx); err != nil {
+		log.Fatalf("joining cluster: %v", err)
+	}
+	defer s.node.Stop(context.Background())
+	log.Printf("instance %s (%s) advertising %s", s.node.ID, s.node.Hostname, s.node.Address)
+
+	// One hub per game. Each hub campaigns for leadership: the winner drives
+	// the rounds and publishes frames, the rest relay those frames to their
+	// own clients. Roles are renegotiated continuously, so losing an instance
+	// hands its rooms over without intervention.
 	for _, g := range games {
-		r := room.New(g, pool, s.ledger)
-		s.rooms[g] = r
-		go func(r *room.Room) {
-			if err := r.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
-				log.Printf("room %s stopped: %v", r.Game, err)
-			}
-		}(r)
+		h := newGameHub(g, room.New(g, pool, s.ledger), s.node)
+		s.rooms[g] = h.room
+		s.hubs[g] = h
+		go h.supervise(ctx)
 	}
 
 	srv := &http.Server{
@@ -147,6 +178,7 @@ func (s *server) routes() http.Handler {
 		mux.HandleFunc("POST /api/dev/faucet", s.handleFaucet)
 	}
 	mux.HandleFunc("GET /ws/{game}", s.handleWS)
+	mux.HandleFunc("GET /api/cluster", s.handleCluster)
 
 	sub, err := fs.Sub(staticFiles, "static")
 	if err != nil {
@@ -179,16 +211,28 @@ func writeErr(w http.ResponseWriter, status int, msg string) {
 	writeJSON(w, status, map[string]string{"error": msg})
 }
 
+// SessionTTL bounds how long a token stays valid without use.
+const SessionTTL = 24 * time.Hour
+
 // session resolves the caller's public key from the Authorization header.
 func (s *server) session(r *http.Request) (string, bool) {
-	token := r.Header.Get("Authorization")
-	if len(token) > 7 && token[:7] == "Bearer " {
-		token = token[7:]
+	token := bearer(r)
+	if token == "" {
+		return "", false
 	}
-	s.sessMu.RLock()
-	defer s.sessMu.RUnlock()
-	pk, ok := s.sessions[token]
-	return pk, ok
+	pk, err := s.rdb.Get(r.Context(), "qa:session:"+token).Result()
+	if err != nil {
+		return "", false
+	}
+	return pk, true
+}
+
+func bearer(r *http.Request) string {
+	token := r.Header.Get("Authorization")
+	if len(token) > 7 && strings.EqualFold(token[:7], "Bearer ") {
+		return token[7:]
+	}
+	return ""
 }
 
 // account resolves the caller to a ledger account id.
@@ -276,9 +320,10 @@ func (s *server) handleVerify(w http.ResponseWriter, r *http.Request) {
 	tokenBytes = seed.Bytes()
 	token := hex.EncodeToString(tokenBytes[:])
 
-	s.sessMu.Lock()
-	s.sessions[token] = req.Pubkey
-	s.sessMu.Unlock()
+	if err := s.rdb.Set(r.Context(), "qa:session:"+token, req.Pubkey, SessionTTL).Err(); err != nil {
+		writeErr(w, http.StatusInternalServerError, "could not store session")
+		return
+	}
 
 	bal, _ := s.ledger.Balance(r.Context(), accountID)
 	writeJSON(w, http.StatusOK, map[string]any{
@@ -348,9 +393,23 @@ func (s *server) handleTransfer(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *server) handleGames(w http.ResponseWriter, r *http.Request) {
-	out := make([]room.Snapshot, 0, len(s.rooms))
+	// Serve the last frame each hub saw rather than the local room object:
+	// on an instance that does not lead a game, the local room is idle and
+	// would report a game that never starts.
+	out := make([]json.RawMessage, 0, len(games))
 	for _, g := range games {
-		out = append(out, s.rooms[g].Snapshot())
+		hub := s.hubs[g]
+		if frame, ok := hub.LastFrame(); ok {
+			out = append(out, json.RawMessage(frame))
+			continue
+		}
+		// Nothing seen yet — fall back to the local view, which is correct
+		// during the moment before the first frame arrives.
+		snap, err := json.Marshal(s.rooms[g].Snapshot())
+		if err != nil {
+			continue
+		}
+		out = append(out, json.RawMessage(snap))
 	}
 	writeJSON(w, http.StatusOK, map[string]any{"rooms": out})
 }
@@ -361,6 +420,11 @@ func (s *server) handleBet(w http.ResponseWriter, r *http.Request) {
 		writeErr(w, http.StatusUnauthorized, "not signed in")
 		return
 	}
+	body, err := readBody(r)
+	if err != nil {
+		writeErr(w, http.StatusBadRequest, "could not read request")
+		return
+	}
 	var req struct {
 		Game      string `json:"game"`
 		StakeMsat int64  `json:"stake_msat"`
@@ -369,7 +433,7 @@ func (s *server) handleBet(w http.ResponseWriter, r *http.Request) {
 		// Zero or absent means no target.
 		AutoCashOut float64 `json:"auto_cashout"`
 	}
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+	if err := json.Unmarshal(body, &req); err != nil {
 		writeErr(w, http.StatusBadRequest, "malformed request")
 		return
 	}
@@ -378,6 +442,10 @@ func (s *server) handleBet(w http.ResponseWriter, r *http.Request) {
 		writeErr(w, http.StatusNotFound, "no such game")
 		return
 	}
+	// Only the instance driving this game holds the authoritative round.
+	if s.forwardToLeader(w, r, req.Game, body) {
+		return
+	}
 	// Convert the target to fixed-point at the boundary; everything past this
 	// point is integer arithmetic.
 	var target fixed.F
@@ -403,10 +471,15 @@ func (s *server) handleCashout(w http.ResponseWriter, r *http.Request) {
 		writeErr(w, http.StatusUnauthorized, "not signed in")
 		return
 	}
+	body, err := readBody(r)
+	if err != nil {
+		writeErr(w, http.StatusBadRequest, "could not read request")
+		return
+	}
 	var req struct {
 		Game string `json:"game"`
 	}
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+	if err := json.Unmarshal(body, &req); err != nil {
 		writeErr(w, http.StatusBadRequest, "malformed request")
 		return
 	}
@@ -415,6 +488,9 @@ func (s *server) handleCashout(w http.ResponseWriter, r *http.Request) {
 		writeErr(w, http.StatusNotFound, "no such game")
 		return
 	}
+	if s.forwardToLeader(w, r, req.Game, body) {
+		return
+	}
 	at, err := rm.CashOut(id)
 	if err != nil {
 		writeErr(w, http.StatusBadRequest, err.Error())
@@ -480,10 +556,12 @@ func (s *server) handleScratchPlay(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	s.nonceMu.Lock()
-	s.scratchNonce++
-	nonce := s.scratchNonce
-	s.nonceMu.Unlock()
+	n, err := s.rdb.Incr(r.Context(), "qa:scratch:nonce").Result()
+	if err != nil {
+		writeErr(w, http.StatusServiceUnavailable, "could not allocate a nonce")
+		return
+	}
+	nonce := uint64(n)
 
 	server := fair.NewServerSeed()
 	outcome, proof := scratch.PlayFromRound(ticket, server, pk, nonce, req.StakeMsat)
@@ -525,6 +603,36 @@ func (s *server) handleScratchPlay(w http.ResponseWriter, r *http.Request) {
 	})
 }
 
+// handleCluster reports the instances currently serving and which of them
+// drives each game. This is the operator's view of a cloned fleet.
+func (s *server) handleCluster(w http.ResponseWriter, r *http.Request) {
+	members, err := s.node.Members(r.Context())
+	if err != nil {
+		writeErr(w, http.StatusServiceUnavailable, err.Error())
+		return
+	}
+	leaders := make(map[string]any, len(games))
+	for _, g := range games {
+		m, err := s.node.LeaderOf(r.Context(), g)
+		if err != nil {
+			continue
+		}
+		leaders[g] = map[string]any{
+			"instance": m.ID,
+			"hostname": m.Hostname,
+			"address":  m.Address,
+			"is_me":    m.ID == s.node.ID,
+		}
+	}
+	writeJSON(w, http.StatusOK, map[string]any{
+		"this_instance": map[string]string{
+			"id": s.node.ID, "hostname": s.node.Hostname, "address": s.node.Address,
+		},
+		"members": members,
+		"leaders": leaders,
+	})
+}
+
 // handleFaucet credits the caller from the bridge account. Development only.
 func (s *server) handleFaucet(w http.ResponseWriter, r *http.Request) {
 	id, _, ok := s.account(r)
@@ -608,11 +716,12 @@ func (s *server) handleVerifyRound(w http.ResponseWriter, r *http.Request) {
 // handleWS streams round snapshots to a client.
 func (s *server) handleWS(w http.ResponseWriter, r *http.Request) {
 	game := r.PathValue("game")
-	rm, ok := s.rooms[game]
+	hub, ok := s.hubs[game]
 	if !ok {
 		writeErr(w, http.StatusNotFound, "no such game")
 		return
 	}
+	rm := hub.room
 
 	// The server is LAN-only, so any origin on the local network is acceptable.
 	conn, err := websocket.Accept(w, r, &websocket.AcceptOptions{
@@ -624,7 +733,9 @@ func (s *server) handleWS(w http.ResponseWriter, r *http.Request) {
 	defer conn.CloseNow()
 
 	ctx := r.Context()
-	updates, unsubscribe := rm.Subscribe()
+	// Frames come from the hub, which produces them when this instance leads
+	// the game and relays the leader's when it does not.
+	updates, unsubscribe := hub.Subscribe()
 	defer unsubscribe()
 
 	// Send the current state immediately so a joining phone is never blank.