casino/pkg/cluster/cluster_test.go

package cluster_test

import (
	"context"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/drjones/quantum-arcade/pkg/cluster"
	"github.com/redis/go-redis/v9"
)

// These tests simulate several cloned instances against one Redis, which is
// exactly the deployment shape: identical VMs, shared coordination.

func testRedis(t *testing.T) *redis.Client {
	t.Helper()
	addr := os.Getenv("ARCADE_TEST_REDIS")
	if addr == "" {
		addr = "localhost:6379"
	}
	rdb := redis.NewClient(&redis.Options{Addr: addr})
	if err := rdb.Ping(context.Background()).Err(); err != nil {
		t.Skipf("no redis available: %v", err)
	}
	return rdb
}

// room returns a name unique to this test run, so parallel packages and repeat
// runs do not fight over the same leadership key.
func room(t *testing.T) string {
	t.Helper()
	return fmt.Sprintf("test-%s-%d", t.Name(), time.Now().UnixNano())
}

func newNode(t *testing.T, rdb *redis.Client, addr string) *cluster.Node {
	t.Helper()
	n := cluster.NewNode(rdb, addr)
	ctx := context.Background()
	if err := n.Start(ctx); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { n.Stop(context.Background()) })
	return n
}

// Two clones of the same image must not end up with the same identity.
func TestClonedInstancesGetDistinctIdentities(t *testing.T) {
	rdb := testRedis(t)
	seen := map[string]bool{}
	for i := 0; i < 200; i++ {
		n := cluster.NewNode(rdb, "10.0.0.1:8080")
		if seen[n.ID] {
			t.Fatalf("identity collision after %d instances: %s", i, n.ID)
		}
		seen[n.ID] = true
	}
}

func TestInstancesDiscoverEachOther(t *testing.T) {
	rdb := testRedis(t)
	ctx := context.Background()

	a := newNode(t, rdb, "10.0.0.1:8080")
	b := newNode(t, rdb, "10.0.0.2:8080")

	members, err := a.Members(ctx)
	if err != nil {
		t.Fatal(err)
	}
	ids := map[string]bool{}
	for _, m := range members {
		ids[m.ID] = true
	}
	if !ids[a.ID] || !ids[b.ID] {
		t.Fatalf("instances did not see each other: %+v", members)
	}
}

// The core guarantee: exactly one instance drives a room, however many
// campaign at once.
func TestExactlyOneLeaderPerRoom(t *testing.T) {
	rdb := testRedis(t)
	ctx := context.Background()
	rm := room(t)

	const instances = 12
	nodes := make([]*cluster.Node, instances)
	for i := range nodes {
		nodes[i] = newNode(t, rdb, fmt.Sprintf("10.0.0.%d:8080", i+1))
	}

	results := make([]bool, instances)
	done := make(chan struct{})
	for i, n := range nodes {
		go func(i int, n *cluster.Node) {
			won, err := n.Campaign(ctx, rm)
			if err != nil {
				t.Errorf("campaign: %v", err)
			}
			results[i] = won
			done <- struct{}{}
		}(i, n)
	}
	for range nodes {
		<-done
	}

	leaders := 0
	for _, won := range results {
		if won {
			leaders++
		}
	}
	if leaders != 1 {
		t.Fatalf("%d instances claimed leadership of one room, want 1", leaders)
	}
}

func TestFollowerFindsTheLeaderAddress(t *testing.T) {
	rdb := testRedis(t)
	ctx := context.Background()
	rm := room(t)

	leader := newNode(t, rdb, "10.0.0.9:8080")
	follower := newNode(t, rdb, "10.0.0.10:8080")

	won, err := leader.Campaign(ctx, rm)
	if err != nil || !won {
		t.Fatalf("leader failed to take the room: won=%v err=%v", won, err)
	}

	m, err := follower.LeaderOf(ctx, rm)
	if err != nil {
		t.Fatal(err)
	}
	if m.ID != leader.ID {
		t.Fatalf("follower found leader %q, want %q", m.ID, leader.ID)
	}
	if m.Address != "10.0.0.9:8080" {
		t.Fatalf("leader address = %q, want 10.0.0.9:8080", m.Address)
	}
}

// Losing the lease must be visible to the instance that lost it, so it stops
// driving the room rather than producing a second stream of rounds.
func TestRenewFailsAfterLeadershipIsLost(t *testing.T) {
	rdb := testRedis(t)
	ctx := context.Background()
	rm := room(t)

	a := newNode(t, rdb, "10.0.0.1:8080")
	b := newNode(t, rdb, "10.0.0.2:8080")

	if won, _ := a.Campaign(ctx, rm); !won {
		t.Fatal("first instance did not win an uncontested room")
	}
	if held, _ := a.Renew(ctx, rm); !held {
		t.Fatal("leader could not renew its own lease")
	}

	// Simulate the lease expiring and another instance taking over.
	if err := a.Resign(ctx, rm); err != nil {
		t.Fatal(err)
	}
	if won, _ := b.Campaign(ctx, rm); !won {
		t.Fatal("second instance could not take the vacated room")
	}

	held, err := a.Renew(ctx, rm)
	if err != nil {
		t.Fatal(err)
	}
	if held {
		t.Fatal("the former leader renewed a lease it no longer holds")
	}
	if a.Leads(rm) {
		t.Fatal("the former leader still believes it leads the room")
	}
}

// A dead instance must hand its room over on its own, without intervention.
func TestLeadershipPassesOnWhenAnInstanceDies(t *testing.T) {
	rdb := testRedis(t)
	ctx := context.Background()
	rm := room(t)

	dying := newNode(t, rdb, "10.0.0.1:8080")
	survivor := newNode(t, rdb, "10.0.0.2:8080")

	if won, _ := dying.Campaign(ctx, rm); !won {
		t.Fatal("first instance did not take the room")
	}
	if won, _ := survivor.Campaign(ctx, rm); won {
		t.Fatal("a second instance took a room that was already led")
	}

	// The instance vanishes without resigning; its lease simply stops being
	// renewed. Waiting out the TTL is the whole point of the mechanism.
	dying.Stop(ctx)
	_ = rdb.Del(ctx, "qa:leader:"+rm) // stand in for the lease expiring

	deadline := time.Now().Add(cluster.LeaseTTL + 3*time.Second)
	took := false
	for time.Now().Before(deadline) {
		if won, _ := survivor.Campaign(ctx, rm); won {
			took = true
			break
		}
		time.Sleep(200 * time.Millisecond)
	}
	if !took {
		t.Fatal("no instance took over the room after the leader died")
	}
}

// A resigning instance must not be able to release a lease that has since
// passed to someone else.
func TestResignDoesNotStealAnotherLease(t *testing.T) {
	rdb := testRedis(t)
	ctx := context.Background()
	rm := room(t)

	a := newNode(t, rdb, "10.0.0.1:8080")
	b := newNode(t, rdb, "10.0.0.2:8080")

	if won, _ := a.Campaign(ctx, rm); !won {
		t.Fatal("a did not take the room")
	}
	if err := a.Resign(ctx, rm); err != nil {
		t.Fatal(err)
	}
	if won, _ := b.Campaign(ctx, rm); !won {
		t.Fatal("b could not take the vacated room")
	}

	// A stale resign from the former leader must be a no-op.
	if err := a.Resign(ctx, rm); err != nil {
		t.Fatal(err)
	}
	m, err := b.LeaderOf(ctx, rm)
	if err != nil {
		t.Fatal(err)
	}
	if m.ID != b.ID {
		t.Fatalf("stale resign released the current leader's lease (leader now %q)", m.ID)
	}
}

// Frames published by the leader must reach the instances relaying them.
func TestFramesFanOutToFollowers(t *testing.T) {
	rdb := testRedis(t)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	rm := room(t)

	leader := newNode(t, rdb, "10.0.0.1:8080")
	follower := newNode(t, rdb, "10.0.0.2:8080")

	frames, unsubscribe := follower.SubscribeFrames(ctx, rm)
	defer unsubscribe()

	// Give the subscription a moment to establish before publishing.
	time.Sleep(300 * time.Millisecond)

	want := []byte(`{"state":"running","multiplier":"2.500000"}`)
	if err := leader.PublishFrame(ctx, rm, want); err != nil {
		t.Fatal(err)
	}

	select {
	case got := <-frames:
		if string(got) != string(want) {
			t.Fatalf("relayed frame = %s, want %s", got, want)
		}
	case <-time.After(4 * time.Second):
		t.Fatal("follower never received the leader's frame")
	}
}

// Instances that stop heartbeating must drop off the roster.
func TestDeadInstancesLeaveTheRoster(t *testing.T) {
	rdb := testRedis(t)
	ctx := context.Background()

	alive := newNode(t, rdb, "10.0.0.1:8080")
	transient := newNode(t, rdb, "10.0.0.2:8080")

	transient.Stop(ctx) // a clean shutdown deregisters immediately

	members, err := alive.Members(ctx)
	if err != nil {
		t.Fatal(err)
	}
	for _, m := range members {
		if m.ID == transient.ID {
			t.Fatal("a stopped instance is still listed as a member")
		}
	}
}