feat: refund abandoned rounds; full-journey and capacity tests
Fixes the money bug flagged earlier. When an instance died mid-round its players had already been debited, so their stakes sat with the house: balanced books, quietly robbed players. Every instance now sweeps for unresolved rounds and refunds them. Such a round is marked void, not settled. The schema caught this: the reveal_is_complete constraint requires a settled round to publish its seed, and an abandoned round has no outcome to reveal. Void is a distinct state with its own column and a check that the two are exclusive. Claiming happens before money moves, so concurrent reconcilers on different instances refund exactly once. Adds TestFullPlayerJourney: sign-in with no account, fund, scratch, bet with an auto target, settle, verify the round independently, check the ledger history is continuous, transfer to a friend, and confirm the books still sum to zero. It asserts against the ledger rather than the API's own summary. Adds cmd/loadtest. One instance on 4 cores held 25,000 concurrent websocket connections with zero failures at 586MB RSS, about 26KB per connection, with the load generator competing for the same CPU. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
drjones
192
pkg/room/reconcile.go
Normal file
192
pkg/room/reconcile.go
Normal file
@@ -0,0 +1,192 @@
|
||||
package room
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/drjones/quantum-arcade/pkg/ledger"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Reconciler refunds rounds that were abandoned mid-flight.
|
||||
//
|
||||
// Stakes are debited when a bet is placed, so if the instance driving a game
|
||||
// dies between the bet and settlement, those stakes sit with the house and the
|
||||
// round never resolves. The books stay balanced — nothing is created or lost —
|
||||
// but the players are quietly short, which is the same thing as being robbed
|
||||
// by accident.
|
||||
//
|
||||
// This finds those rounds and refunds every unsettled stake. It is safe to run
|
||||
// repeatedly and from any instance: refunds are recorded against the round, and
|
||||
// a round already marked settled is skipped.
|
||||
type Reconciler struct {
|
||||
pool *pgxpool.Pool
|
||||
ledger *ledger.Ledger
|
||||
|
||||
// Stale is how long a round may remain unsettled before it is considered
|
||||
// abandoned. It must exceed the longest possible round plus the time it
|
||||
// takes another instance to take over, or a live round would be refunded
|
||||
// out from under the players still in it.
|
||||
Stale time.Duration
|
||||
}
|
||||
|
||||
func NewReconciler(pool *pgxpool.Pool, l *ledger.Ledger) *Reconciler {
|
||||
return &Reconciler{
|
||||
pool: pool,
|
||||
ledger: l,
|
||||
// A round is capped at 60s of flight plus its betting and settle
|
||||
// phases; leadership moves within LeaseTTL. Two minutes is far past
|
||||
// any legitimate round and still prompt enough to matter at a party.
|
||||
Stale: 2 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
// Result describes what a reconciliation pass did.
|
||||
type Result struct {
|
||||
RoundsRefunded int
|
||||
BetsRefunded int
|
||||
MsatRefunded int64
|
||||
}
|
||||
|
||||
// Run refunds every abandoned round it finds.
|
||||
func (rc *Reconciler) Run(ctx context.Context) (Result, error) {
|
||||
var res Result
|
||||
|
||||
rows, err := rc.pool.Query(ctx, `
|
||||
SELECT DISTINCT r.id
|
||||
FROM rounds r
|
||||
JOIN bets b ON b.round_id = r.id
|
||||
WHERE r.settled_at IS NULL
|
||||
AND r.voided_at IS NULL
|
||||
AND b.settled_at IS NULL
|
||||
AND r.opened_at < now() - make_interval(secs => $1)
|
||||
ORDER BY r.id`,
|
||||
rc.Stale.Seconds())
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("finding abandoned rounds: %w", err)
|
||||
}
|
||||
|
||||
var roundIDs []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return res, err
|
||||
}
|
||||
roundIDs = append(roundIDs, id)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
for _, roundID := range roundIDs {
|
||||
refunded, msat, err := rc.refundRound(ctx, roundID)
|
||||
if err != nil {
|
||||
// One bad round must not stop the rest from being made whole.
|
||||
fmt.Printf("reconcile: round %d: %v\n", roundID, err)
|
||||
continue
|
||||
}
|
||||
if refunded > 0 {
|
||||
res.RoundsRefunded++
|
||||
res.BetsRefunded += refunded
|
||||
res.MsatRefunded += msat
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// refundRound returns every unsettled stake in one round.
|
||||
func (rc *Reconciler) refundRound(ctx context.Context, roundID int64) (int, int64, error) {
|
||||
// Claim the round by voiding it. Doing this before moving money means a
|
||||
// second pass — or another instance running concurrently — finds nothing
|
||||
// to do, so a refund cannot be issued twice.
|
||||
//
|
||||
// Void, not settled: an abandoned round produced no outcome, so it has no
|
||||
// seed to reveal and must not masquerade as a resolved round.
|
||||
tag, err := rc.pool.Exec(ctx,
|
||||
`UPDATE rounds SET voided_at = now()
|
||||
WHERE id = $1 AND settled_at IS NULL AND voided_at IS NULL`, roundID)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("claiming round: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return 0, 0, nil // another pass got there first
|
||||
}
|
||||
|
||||
rows, err := rc.pool.Query(ctx,
|
||||
`SELECT account_id, stake_msat FROM bets
|
||||
WHERE round_id = $1 AND settled_at IS NULL`, roundID)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
type refund struct {
|
||||
account int64
|
||||
msat int64
|
||||
}
|
||||
var refunds []refund
|
||||
for rows.Next() {
|
||||
var r refund
|
||||
if err := rows.Scan(&r.account, &r.msat); err != nil {
|
||||
rows.Close()
|
||||
return 0, 0, err
|
||||
}
|
||||
refunds = append(refunds, r)
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if len(refunds) == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
house, err := rc.ledger.AccountByName(ctx, "house_pot")
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
postings := make([]ledger.Posting, 0, len(refunds)+1)
|
||||
var total int64
|
||||
for _, r := range refunds {
|
||||
postings = append(postings, ledger.Posting{AccountID: r.account, AmountMsat: r.msat})
|
||||
total += r.msat
|
||||
}
|
||||
postings = append(postings, ledger.Posting{AccountID: house, AmountMsat: -total})
|
||||
|
||||
rid := roundID
|
||||
if _, err := rc.ledger.Post(ctx, "refund_abandoned", &rid, postings); err != nil {
|
||||
return 0, 0, fmt.Errorf("posting refunds: %w", err)
|
||||
}
|
||||
|
||||
if _, err := rc.pool.Exec(ctx,
|
||||
`UPDATE bets SET settled_at = now(), payout_msat = stake_msat
|
||||
WHERE round_id = $1 AND settled_at IS NULL`, roundID); err != nil {
|
||||
return 0, 0, fmt.Errorf("marking bets refunded: %w", err)
|
||||
}
|
||||
|
||||
return len(refunds), total, nil
|
||||
}
|
||||
|
||||
// RunPeriodically sweeps for abandoned rounds until the context ends.
|
||||
func (rc *Reconciler) RunPeriodically(ctx context.Context, every time.Duration) {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
res, err := rc.Run(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("reconcile: %v\n", err)
|
||||
continue
|
||||
}
|
||||
if res.RoundsRefunded > 0 {
|
||||
fmt.Printf("reconcile: refunded %d bets across %d abandoned rounds (%d msat)\n",
|
||||
res.BetsRefunded, res.RoundsRefunded, res.MsatRefunded)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -814,3 +814,185 @@ func TestManualCashOutOverridesAPendingTarget(t *testing.T) {
|
||||
t.Fatalf("manual cash-out returned %v, expected the current multiplier", at)
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------------- abandoned round reconciliation ---------------- */
|
||||
|
||||
// The scenario: an instance takes bets, then dies before settling. The stakes
|
||||
// have already left the players' balances. Nobody should be quietly short.
|
||||
func TestAbandonedRoundIsRefunded(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
rc := NewReconciler(f.room.pool, f.ledger)
|
||||
rc.Stale = 0 // treat everything as abandoned, for the test
|
||||
|
||||
id, pk := f.player("a", 100_000)
|
||||
before, _ := f.ledger.Balance(f.ctx, id)
|
||||
|
||||
f.openBetting()
|
||||
const stake = 10_000
|
||||
if err := f.room.PlaceBet(f.ctx, id, pk, "a", stake, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterBet, _ := f.ledger.Balance(f.ctx, id)
|
||||
if before-afterBet != stake {
|
||||
t.Fatalf("stake not taken: %d", before-afterBet)
|
||||
}
|
||||
|
||||
// The instance dies here: the round is never settled.
|
||||
res, err := rc.Run(f.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.BetsRefunded < 1 {
|
||||
t.Fatalf("nothing refunded: %+v", res)
|
||||
}
|
||||
|
||||
afterRefund, _ := f.ledger.Balance(f.ctx, id)
|
||||
if afterRefund != before {
|
||||
t.Fatalf("balance = %d after refund, want %d (the original stake back)",
|
||||
afterRefund, before)
|
||||
}
|
||||
}
|
||||
|
||||
// Running twice must not pay twice.
|
||||
func TestReconcileIsIdempotent(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
rc := NewReconciler(f.room.pool, f.ledger)
|
||||
rc.Stale = 0
|
||||
|
||||
id, pk := f.player("a", 100_000)
|
||||
before, _ := f.ledger.Balance(f.ctx, id)
|
||||
|
||||
f.openBetting()
|
||||
if err := f.room.PlaceBet(f.ctx, id, pk, "a", 10_000, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := rc.Run(f.ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterFirst, _ := f.ledger.Balance(f.ctx, id)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := rc.Run(f.ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
afterRepeats, _ := f.ledger.Balance(f.ctx, id)
|
||||
|
||||
if afterRepeats != afterFirst {
|
||||
t.Fatalf("repeated reconciliation paid again: %d -> %d", afterFirst, afterRepeats)
|
||||
}
|
||||
if afterFirst != before {
|
||||
t.Fatalf("refund was not exactly the stake: %d, want %d", afterFirst, before)
|
||||
}
|
||||
}
|
||||
|
||||
// Concurrent reconcilers, as two instances would be, must still refund once.
|
||||
func TestConcurrentReconcilersRefundOnce(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
rc1 := NewReconciler(f.room.pool, f.ledger)
|
||||
rc2 := NewReconciler(f.room.pool, f.ledger)
|
||||
rc1.Stale, rc2.Stale = 0, 0
|
||||
|
||||
id, pk := f.player("a", 100_000)
|
||||
before, _ := f.ledger.Balance(f.ctx, id)
|
||||
|
||||
f.openBetting()
|
||||
if err := f.room.PlaceBet(f.ctx, id, pk, "a", 10_000, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, rc := range []*Reconciler{rc1, rc2, rc1, rc2} {
|
||||
wg.Add(1)
|
||||
go func(rc *Reconciler) {
|
||||
defer wg.Done()
|
||||
_, _ = rc.Run(f.ctx)
|
||||
}(rc)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
after, _ := f.ledger.Balance(f.ctx, id)
|
||||
if after != before {
|
||||
t.Fatalf("concurrent reconcilers refunded %d, want exactly the stake (%d)",
|
||||
after-before+10_000, 10_000)
|
||||
}
|
||||
}
|
||||
|
||||
// A round that settled normally must never be refunded on top of its payout.
|
||||
func TestSettledRoundIsNotRefunded(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
rc := NewReconciler(f.room.pool, f.ledger)
|
||||
rc.Stale = 0
|
||||
|
||||
id, pk := f.player("a", 100_000)
|
||||
f.openBetting()
|
||||
if err := f.room.PlaceBet(f.ctx, id, pk, "a", 10_000, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.startRun()
|
||||
if err := f.room.settle(f.ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterSettle, _ := f.ledger.Balance(f.ctx, id)
|
||||
|
||||
if _, err := rc.Run(f.ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterReconcile, _ := f.ledger.Balance(f.ctx, id)
|
||||
|
||||
if afterReconcile != afterSettle {
|
||||
t.Fatalf("a settled round was refunded: %d -> %d", afterSettle, afterReconcile)
|
||||
}
|
||||
}
|
||||
|
||||
// A round still in flight must be left alone.
|
||||
func TestLiveRoundIsNotRefunded(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
rc := NewReconciler(f.room.pool, f.ledger)
|
||||
rc.Stale = 2 * time.Minute // the production value
|
||||
|
||||
id, pk := f.player("a", 100_000)
|
||||
f.openBetting()
|
||||
if err := f.room.PlaceBet(f.ctx, id, pk, "a", 10_000, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterBet, _ := f.ledger.Balance(f.ctx, id)
|
||||
|
||||
res, err := rc.Run(f.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.RoundsRefunded != 0 {
|
||||
t.Fatalf("a live round was refunded out from under its players: %+v", res)
|
||||
}
|
||||
after, _ := f.ledger.Balance(f.ctx, id)
|
||||
if after != afterBet {
|
||||
t.Fatalf("balance changed on a live round: %d -> %d", afterBet, after)
|
||||
}
|
||||
}
|
||||
|
||||
// The books must still balance after a refund.
|
||||
func TestBooksBalanceAfterRefund(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
rc := NewReconciler(f.room.pool, f.ledger)
|
||||
rc.Stale = 0
|
||||
|
||||
f.openBetting()
|
||||
for i := 0; i < 5; i++ {
|
||||
id, pk := f.player(fmt.Sprintf("p%d", i), 100_000)
|
||||
if err := f.room.PlaceBet(f.ctx, id, pk, "p", 10_000, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := rc.Run(f.ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
total, err := f.ledger.ConservationCheck(f.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("books do not balance after refunds: %d", total)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user