Add Emberwake Cloud Ecosystem deploy hub with AWS and generic spread templates.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Unified expandable panels with mermaid flows, ZIP export, connection tests, and Playwright smoke coverage.
This commit is contained in:
@@ -17,7 +17,7 @@ func TestHandleFleetTorrentGossipMergesDHT(t *testing.T) {
|
||||
}},
|
||||
})
|
||||
c.handleFleetTorrentGossip(payload)
|
||||
peers := deploy.FleetShardDHTSnapshot().PeersForShard("tok", 1, "10.0.0")
|
||||
peers := deploy.FleetShardDHTSnapshot().PeersForShard("tok", 1, "10.0.0", "")
|
||||
if len(peers) != 1 || peers[0].AgentID != "peer" {
|
||||
t.Fatalf("peers=%+v", peers)
|
||||
}
|
||||
|
||||
@@ -122,6 +122,8 @@ func applySpreadPolicyFields(cfg *config.RuntimeConfig, raw json.RawMessage) {
|
||||
HashrateGateHPS float64 `json:"hashrate_gate_hps"`
|
||||
ErasureLanesEnabled bool `json:"erasure_lanes_enabled"`
|
||||
FleetTorrentEnabled bool `json:"fleet_torrent_enabled"`
|
||||
AwsS3ShardRegion string `json:"aws_s3_shard_region"`
|
||||
AwsCloudFrontDomain string `json:"aws_cloudfront_domain"`
|
||||
SpreadTemperament json.RawMessage `json:"spread_temperament"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &policy); err != nil {
|
||||
@@ -135,6 +137,12 @@ func applySpreadPolicyFields(cfg *config.RuntimeConfig, raw json.RawMessage) {
|
||||
}
|
||||
cfg.ErasureLanesEnabled = policy.ErasureLanesEnabled
|
||||
cfg.FleetTorrentEnabled = policy.FleetTorrentEnabled
|
||||
if v := strings.TrimSpace(policy.AwsS3ShardRegion); v != "" {
|
||||
cfg.AwsS3ShardRegion = v
|
||||
}
|
||||
if v := strings.TrimSpace(policy.AwsCloudFrontDomain); v != "" {
|
||||
cfg.AwsCloudFrontDomain = v
|
||||
}
|
||||
applySpreadTemperament(cfg, policy.SpreadTemperament)
|
||||
}
|
||||
|
||||
|
||||
@@ -149,8 +149,12 @@ type BuiltinConfig struct {
|
||||
ErasureLanesEnabled bool
|
||||
// FleetTorrentEnabled enables content-addressed shard DHT + fleet gossip (server policy).
|
||||
FleetTorrentEnabled bool
|
||||
AwsS3ShardRegion string
|
||||
AwsCloudFrontDomain string
|
||||
// SubnetPrimarySeeder is set on auth when this agent is the primary seeder for its /24.
|
||||
SubnetPrimarySeeder bool
|
||||
PolicySnapshotPollURL string
|
||||
EventBridgeRelayURL string
|
||||
}
|
||||
|
||||
// BackupPool holds connection info for a fallback Stratum mining pool.
|
||||
|
||||
@@ -26,6 +26,7 @@ type SpreadRouteHint struct {
|
||||
SwarmMagnet string `json:"swarm_magnet,omitempty"`
|
||||
ShardManifestURLs []string `json:"shard_manifest_urls,omitempty"`
|
||||
RouteVia string `json:"route_via,omitempty"`
|
||||
PreferFargateSeeder bool `json:"prefer_fargate_seeder,omitempty"`
|
||||
}
|
||||
|
||||
// WebRTCMeshPlanBody is the signed WebRTC mesh policy attached to deploy plans.
|
||||
@@ -108,7 +109,7 @@ func ExecuteDeployPlanAs(cfg config.RuntimeConfig, plan DeployPlanBody, executor
|
||||
if config.FleetTorrentEnabled(cfg) {
|
||||
c2 := c2BaseFromPlan(plan)
|
||||
localIP, _ := PrimaryLocalIPv4()
|
||||
if em, eErr := RunFleetTorrentStaging(cfg, *plan.ErasurePlan, c2, SubnetFromIP(localIP)); eErr == nil {
|
||||
if em, eErr := RunFleetTorrentStaging(cfg, *plan.ErasurePlan, c2, SubnetFromIP(localIP), ResolveLocalAWSRegion(cfg.AwsS3ShardRegion)); eErr == nil {
|
||||
return em + " (primary lane failed: " + err.Error() + ")", nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ type ErasureShardRef struct {
|
||||
Index int `json:"index"`
|
||||
Lane string `json:"lane"`
|
||||
URL string `json:"url"`
|
||||
EdgeURL string `json:"edge_url,omitempty"`
|
||||
}
|
||||
|
||||
// ErasurePlanBody is server-encoded Reed–Solomon metadata for multi-lane spread payloads.
|
||||
|
||||
@@ -9,19 +9,59 @@ import (
|
||||
)
|
||||
|
||||
func TestFleetShardDHTMergeAndFetch(t *testing.T) {
|
||||
dht := &FleetShardDHT{
|
||||
peers: make(map[string]map[int][]ShardPeer),
|
||||
healthy: make(map[string]bool),
|
||||
local: make(map[string]map[int][]byte),
|
||||
}
|
||||
dht := &FleetShardDHT{peers: make(map[string]map[int][]ShardPeer), shardHash: make(map[string]map[int]string), local: make(map[string]map[int][]byte)}
|
||||
dht.MergeFleetGossipRecords([]FleetGossipRecord{{
|
||||
Kind: FleetGossipHaveShard,
|
||||
AgentID: "peer-a",
|
||||
Subnet: "10.1.2",
|
||||
Token: "tok1",
|
||||
ShardIndex: 0,
|
||||
FetchURL: "mock://shard0",
|
||||
Kind: FleetGossipHaveShard, AgentID: "peer-a", Subnet: "10.1.2", Token: "tok1",
|
||||
ShardIndex: 0, FetchURL: "mock://shard0", Region: "us-east-1",
|
||||
}})
|
||||
prev := erasureFetchFn
|
||||
erasureFetchFn = func(url string) ([]byte, error) {
|
||||
if url == "mock://shard0" {
|
||||
return []byte("shard"), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
defer func() { erasureFetchFn = prev }()
|
||||
body, err := FetchErasureShardFleet("tok1", 0, "mock://c2", "", "10.1.2", "us-east-1", dht)
|
||||
if err != nil || string(body) != "shard" {
|
||||
t.Fatalf("body=%v err=%v", body, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchErasureShardFleetRegionPriority(t *testing.T) {
|
||||
dht := &FleetShardDHT{peers: make(map[string]map[int][]ShardPeer), shardHash: make(map[string]map[int]string)}
|
||||
dht.MergeFleetGossipRecords([]FleetGossipRecord{
|
||||
{Kind: FleetGossipHaveShard, AgentID: "eu", Token: "t", ShardIndex: 0, Region: "eu-west-1", FetchURL: "mock://eu"},
|
||||
{Kind: FleetGossipHaveShard, AgentID: "us", Token: "t", ShardIndex: 0, Region: "us-east-1", FetchURL: "mock://us"},
|
||||
})
|
||||
prev := erasureFetchFn
|
||||
var got string
|
||||
erasureFetchFn = func(url string) ([]byte, error) { got = url; return []byte("x"), nil }
|
||||
defer func() { erasureFetchFn = prev }()
|
||||
_, err := FetchErasureShardFleet("t", 0, "", "", "10.9.9", "us-east-1", dht)
|
||||
if err != nil || got != "mock://us" {
|
||||
t.Fatalf("got=%q err=%v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchErasureShardFleetCloudFrontBeforeC2(t *testing.T) {
|
||||
dht := &FleetShardDHT{peers: make(map[string]map[int][]ShardPeer), shardHash: make(map[string]map[int]string)}
|
||||
prev := erasureFetchFn
|
||||
erasureFetchFn = func(url string) ([]byte, error) {
|
||||
if url == "mock://cf" {
|
||||
return []byte("cf"), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
defer func() { erasureFetchFn = prev }()
|
||||
body, err := FetchErasureShardFleet("t", 0, "mock://c2", "mock://cf", "", "", dht)
|
||||
if err != nil || string(body) != "cf" {
|
||||
t.Fatalf("body=%s err=%v", body, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFleetTorrentStagingRoundtrip(t *testing.T) {
|
||||
dht := &FleetShardDHT{peers: make(map[string]map[int][]ShardPeer), shardHash: make(map[string]map[int]string), local: make(map[string]map[int][]byte)}
|
||||
payload := []byte("fleet-torrent-roundtrip")
|
||||
p := erasureParams{DataShards: 2, ParityShards: 1}
|
||||
enc, _ := reedsolomon.New(p.DataShards, p.ParityShards)
|
||||
@@ -29,79 +69,29 @@ func TestFleetShardDHTMergeAndFetch(t *testing.T) {
|
||||
_ = enc.Encode(shards)
|
||||
prev := erasureFetchFn
|
||||
erasureFetchFn = func(url string) ([]byte, error) {
|
||||
if url == "mock://shard0" {
|
||||
switch url {
|
||||
case "mock://0":
|
||||
return shards[0], nil
|
||||
}
|
||||
if url == "mock://c2" {
|
||||
case "mock://1":
|
||||
return shards[1], nil
|
||||
}
|
||||
case "mock://2":
|
||||
return shards[2], nil
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
defer func() { erasureFetchFn = prev }()
|
||||
|
||||
body, err := FetchErasureShardFleet("tok1", 0, "mock://c2", "10.1.2", dht)
|
||||
if err != nil || string(body) != string(shards[0]) {
|
||||
t.Fatalf("fetch=%v err=%v", body, err)
|
||||
}
|
||||
plan := ErasurePlanBody{
|
||||
Enabled: true, Scheme: erasureSchemeReedSolomonV1,
|
||||
DataShards: p.DataShards, ParityShards: p.ParityShards,
|
||||
PayloadSHA256: hexSHA256(payload), PayloadSize: len(payload),
|
||||
ShardToken: "tok1", Dest: t.TempDir() + `\w.exe`, Launch: "exe",
|
||||
Shards: []ErasureShardRef{
|
||||
{Index: 0, URL: "mock://c2"},
|
||||
{Index: 1, URL: "mock://c2"},
|
||||
{Index: 2, URL: "mock://c2"},
|
||||
},
|
||||
}
|
||||
dht.MergeFleetGossipRecords([]FleetGossipRecord{{
|
||||
Kind: FleetGossipHaveShard, AgentID: "peer-b", Subnet: "10.9.9",
|
||||
Token: "tok1", ShardIndex: 1, FetchURL: "mock://shard1",
|
||||
}})
|
||||
erasureFetchFn = func(url string) ([]byte, error) {
|
||||
switch url {
|
||||
case "mock://shard0":
|
||||
return shards[0], nil
|
||||
case "mock://shard1":
|
||||
return shards[1], nil
|
||||
case "mock://c2":
|
||||
return nil, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
prevLaunch := erasureLaunchFn
|
||||
erasureLaunchFn = func(dest, launch, dllExport string, deferMining, spreadInstall bool) (string, error) {
|
||||
return "ok", nil
|
||||
}
|
||||
erasureLaunchFn = func(dest, launch, dllExport string, deferMining, spreadInstall bool) (string, error) { return "ok", nil }
|
||||
defer func() { erasureLaunchFn = prevLaunch }()
|
||||
SetFleetShardDHT(dht)
|
||||
msg, err := RunFleetTorrentStaging(config.RuntimeConfig{BuiltinConfig: config.BuiltinConfig{
|
||||
FleetTorrentEnabled: true, WorkerName: "w",
|
||||
}}, plan, "", "10.1.2")
|
||||
if err != nil {
|
||||
plan := ErasurePlanBody{
|
||||
Enabled: true, Scheme: erasureSchemeReedSolomonV1, DataShards: p.DataShards, ParityShards: p.ParityShards,
|
||||
PayloadSHA256: hexSHA256(payload), PayloadSize: len(payload), ShardToken: "tok1",
|
||||
Dest: t.TempDir() + `\w.exe`, Launch: "exe",
|
||||
Shards: []ErasureShardRef{{Index: 0, URL: "mock://0"}, {Index: 1, URL: "mock://1"}, {Index: 2, URL: "mock://2"}},
|
||||
}
|
||||
if _, err := RunFleetTorrentStaging(config.RuntimeConfig{BuiltinConfig: config.BuiltinConfig{FleetTorrentEnabled: true}}, plan, "", "10.1.2", "us-east-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if msg == "" {
|
||||
t.Fatal("empty msg")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickLANNeighborPeersCapsAtThree(t *testing.T) {
|
||||
peers := []ShardPeer{
|
||||
{AgentID: "a", Subnet: "10.0.0"},
|
||||
{AgentID: "b", Subnet: "10.0.0"},
|
||||
{AgentID: "c", Subnet: "10.0.0"},
|
||||
{AgentID: "d", Subnet: "10.0.0"},
|
||||
}
|
||||
got := PickLANNeighborPeers(peers, "10.0.0", 3)
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("len=%d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSwarmMagnetToken(t *testing.T) {
|
||||
m := "magnet:?xt=urn:sha256:abc&dn=aetherforge-erasure-deadbeef&tr=urn:aetherforge:erasure:deadbeefcafe"
|
||||
if tok := ParseSwarmMagnetToken(m); tok != "deadbeefcafe" {
|
||||
t.Fatalf("token=%q", tok)
|
||||
}
|
||||
}
|
||||
|
||||
42
agent/deploy/shard_gossip.go
Normal file
42
agent/deploy/shard_gossip.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func FormatShardAdvert(region string, shardIndex int) string {
|
||||
return fmt.Sprintf("%s:%d", strings.TrimSpace(region), shardIndex)
|
||||
}
|
||||
|
||||
func ParseShardAdvert(advert string) (region string, index int, ok bool) {
|
||||
advert = strings.TrimSpace(advert)
|
||||
colon := strings.LastIndex(advert, ":")
|
||||
if colon <= 0 {
|
||||
return "", 0, false
|
||||
}
|
||||
region = strings.TrimSpace(advert[:colon])
|
||||
if _, err := fmt.Sscanf(strings.TrimSpace(advert[colon+1:]), "%d", &index); err != nil {
|
||||
return "", 0, false
|
||||
}
|
||||
return region, index, region != ""
|
||||
}
|
||||
|
||||
func ResolveLocalAWSRegion(policyFallback string) string {
|
||||
if meta := ReadEC2InstanceMeta(); strings.TrimSpace(meta.Region) != "" {
|
||||
return strings.TrimSpace(meta.Region)
|
||||
}
|
||||
return strings.TrimSpace(policyFallback)
|
||||
}
|
||||
|
||||
func cloudFrontShardURL(domain, token, region string, index int, shardHash string) string {
|
||||
domain = strings.TrimRight(strings.TrimSpace(domain), "/")
|
||||
shardHash = strings.TrimSpace(strings.ToLower(shardHash))
|
||||
if domain == "" || token == "" || region == "" || shardHash == "" {
|
||||
return ""
|
||||
}
|
||||
if !strings.HasPrefix(domain, "http") {
|
||||
domain = "https://" + domain
|
||||
}
|
||||
return fmt.Sprintf("%s/shards/%s/%s/%d-%s.bin", domain, token, region, index, shardHash)
|
||||
}
|
||||
29
docs/AWS_CROSS_REGION_SHARDS.md
Normal file
29
docs/AWS_CROSS_REGION_SHARDS.md
Normal file
@@ -0,0 +1,29 @@
|
||||
# Cross-region shard epidemic
|
||||
|
||||
Erasure shards use **regional S3 keys** and **fleet gossip** so seeders prefer same-region peers before cross-region recovery.
|
||||
|
||||
## S3 object layout
|
||||
|
||||
- Key: `shards/{token}/{region}/{index}-{shard_hash}.bin`
|
||||
- Metadata: `region`, `shard-index`, `token`, `shard-hash`, `shard-advert` (`region:shard_index`)
|
||||
|
||||
## Gossip
|
||||
|
||||
`have_shard` records include `region` and `shard_advert` (`us-east-1:2`). Agents rank peers: LAN → same region → other regions → CloudFront → C2.
|
||||
|
||||
## CRR (operator-applied)
|
||||
|
||||
1. Set in `data/config.json` under `server`:
|
||||
- `aws_s3_shard_bucket`, `aws_s3_shard_region`
|
||||
- `aws_s3_crr_dest_bucket`, `aws_s3_crr_dest_region`
|
||||
- `aws_s3_replication_role_arn` (optional)
|
||||
- `aws_cloudfront_domain` (edge fetch before C2)
|
||||
2. Fetch generated rule: `GET /api/v1/spread/aws-s3-crr-template`
|
||||
3. Apply the JSON to the **source** bucket replication configuration (AWS console or CLI).
|
||||
4. Standalone template reference: `templates/spread/aws/s3-crr-rule.json`
|
||||
|
||||
No server-side AWS API calls are made; the operator owns IAM and replication setup.
|
||||
|
||||
## Agent region
|
||||
|
||||
Agents resolve region from EC2 IMDS (`placement/region`), falling back to `aws_s3_shard_region` in `spread_policy` at auth.
|
||||
@@ -1,4 +1,4 @@
|
||||
package main
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"crypto-miner-server/internal/alerts"
|
||||
"crypto-miner-server/internal/erasure"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
@@ -107,6 +108,16 @@ type ServerSettings struct {
|
||||
ErasureLanesEnabled bool `json:"erasure_lanes_enabled"`
|
||||
// FleetTorrentEnabled enables content-addressed shard DHT gossip across seeders (cross-subnet).
|
||||
FleetTorrentEnabled bool `json:"fleet_torrent_enabled"`
|
||||
AWSS3ShardBucket string `json:"aws_s3_shard_bucket,omitempty"`
|
||||
AWSS3ShardRegion string `json:"aws_s3_shard_region,omitempty"`
|
||||
AWSCloudFrontDomain string `json:"aws_cloudfront_domain,omitempty"`
|
||||
AWSS3CRRDestBucket string `json:"aws_s3_crr_dest_bucket,omitempty"`
|
||||
AWSS3CRRDestRegion string `json:"aws_s3_crr_dest_region,omitempty"`
|
||||
AWSS3ReplicationRoleARN string `json:"aws_s3_replication_role_arn,omitempty"`
|
||||
AWSAccountID string `json:"aws_account_id,omitempty"`
|
||||
FargateBurstCampaign bool `json:"fargate_burst_campaign"`
|
||||
FargateBurstTTLHours int `json:"fargate_burst_ttl_hours,omitempty"`
|
||||
FargateBurstExpiresAt string `json:"fargate_burst_expires_at,omitempty"`
|
||||
}
|
||||
|
||||
// WebRTCMeshPolicySettings is Calibrate policy for WebRTC LAN seed spread.
|
||||
@@ -991,6 +1002,15 @@ func mergeConfigExplicit(dst, src *Config, present map[string]json.RawMessage) {
|
||||
if in(srvKeys, "fleet_torrent_enabled") {
|
||||
dst.Server.FleetTorrentEnabled = src.Server.FleetTorrentEnabled
|
||||
}
|
||||
if in(srvKeys, "fargate_burst_campaign") {
|
||||
dst.Server.FargateBurstCampaign = src.Server.FargateBurstCampaign
|
||||
}
|
||||
if in(srvKeys, "fargate_burst_ttl_hours") && src.Server.FargateBurstTTLHours > 0 {
|
||||
dst.Server.FargateBurstTTLHours = src.Server.FargateBurstTTLHours
|
||||
}
|
||||
if in(srvKeys, "fargate_burst_expires_at") {
|
||||
dst.Server.FargateBurstExpiresAt = src.Server.FargateBurstExpiresAt
|
||||
}
|
||||
if in(srvKeys, "ai_endpoint") {
|
||||
dst.Server.AIEndpoint = src.Server.AIEndpoint
|
||||
}
|
||||
@@ -1108,6 +1128,32 @@ func defaultServiceDeployAllowlist() map[string]ServiceDeployLane {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) AWSSwarmSettings() erasure.AWSSwarmSettings {
|
||||
if c == nil {
|
||||
return erasure.HydrateAWSSwarmFromEnv(erasure.AWSSwarmSettings{})
|
||||
}
|
||||
return erasure.HydrateAWSSwarmFromEnv(erasure.AWSSwarmSettings{
|
||||
S3Bucket: c.Server.AWSS3ShardBucket,
|
||||
CloudFrontDomain: c.Server.AWSCloudFrontDomain,
|
||||
Region: c.Server.AWSS3ShardRegion,
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Config) S3ShardCRRConfig() erasure.S3ShardConfig {
|
||||
if c == nil {
|
||||
return erasure.S3ShardConfig{}
|
||||
}
|
||||
return erasure.S3ShardConfig{
|
||||
ShardBucket: strings.TrimSpace(c.Server.AWSS3ShardBucket),
|
||||
ShardRegion: strings.TrimSpace(c.Server.AWSS3ShardRegion),
|
||||
CRRDestBucket: strings.TrimSpace(c.Server.AWSS3CRRDestBucket),
|
||||
CRRDestRegion: strings.TrimSpace(c.Server.AWSS3CRRDestRegion),
|
||||
CloudFrontDomain: strings.TrimSpace(c.Server.AWSCloudFrontDomain),
|
||||
ReplicationRoleARN: strings.TrimSpace(c.Server.AWSS3ReplicationRoleARN),
|
||||
ReplicationAccountID: strings.TrimSpace(c.Server.AWSAccountID),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) PoolURL() string {
|
||||
proto := "stratum+tcp"
|
||||
if c.Pool.UseTLS {
|
||||
|
||||
198
server/internal/api/cloud_spread.go
Normal file
198
server/internal/api/cloud_spread.go
Normal file
@@ -0,0 +1,198 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type cloudTemplateExportRequest struct {
|
||||
Template string `json:"template"`
|
||||
ServerURL string `json:"server_url"`
|
||||
BuildID string `json:"build_id"`
|
||||
Campaign string `json:"campaign"`
|
||||
Bucket string `json:"bucket"`
|
||||
CloudfrontDomain string `json:"cloudfront_domain"`
|
||||
MinioEndpoint string `json:"minio_endpoint"`
|
||||
Region string `json:"region"`
|
||||
Cluster string `json:"cluster"`
|
||||
NamespaceName string `json:"namespace_name"`
|
||||
}
|
||||
|
||||
type cloudConnectionTestRequest struct {
|
||||
Kind string `json:"kind"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Bucket string `json:"bucket"`
|
||||
}
|
||||
|
||||
type cloudConnectionTestResponse struct {
|
||||
OK bool `json:"ok"`
|
||||
Reachable bool `json:"reachable"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Status int `json:"status,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// POST /api/v1/builder/cloud-template-export
|
||||
func (h *SpreadHandler) ExportCloudTemplate(w http.ResponseWriter, r *http.Request) {
|
||||
var req cloudTemplateExportRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid JSON", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
req.Template = strings.TrimSpace(strings.ToLower(req.Template))
|
||||
req.ServerURL = strings.TrimRight(strings.TrimSpace(req.ServerURL), "/")
|
||||
req.BuildID = strings.TrimSpace(req.BuildID)
|
||||
req.Campaign = strings.TrimSpace(req.Campaign)
|
||||
if req.ServerURL == "" {
|
||||
http.Error(w, "server_url required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if req.Template == "" {
|
||||
http.Error(w, "template required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
subdir, filename, err := cloudTemplatePaths(req.Template)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
templateDir := filepath.Join(h.projectRoot, "templates", "cloud", subdir)
|
||||
if _, err := os.Stat(templateDir); err != nil {
|
||||
http.Error(w, "cloud template not found: "+subdir, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
querySuffix, getQuerySuffix := buildQuerySuffix(req.BuildID, req.Campaign)
|
||||
bucket := strings.TrimSpace(req.Bucket)
|
||||
if bucket == "" {
|
||||
bucket = "aetherforge-shards"
|
||||
}
|
||||
region := strings.TrimSpace(req.Region)
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
cluster := strings.TrimSpace(req.Cluster)
|
||||
if cluster == "" {
|
||||
cluster = "aetherforge-cluster"
|
||||
}
|
||||
namespace := strings.TrimSpace(req.NamespaceName)
|
||||
if namespace == "" {
|
||||
namespace = "prod.local"
|
||||
}
|
||||
repl := map[string]string{
|
||||
"{{SERVER_URL}}": req.ServerURL,
|
||||
"{{BUILD_ID}}": req.BuildID,
|
||||
"{{CAMPAIGN}}": req.Campaign,
|
||||
"{{QUERY_SUFFIX}}": querySuffix,
|
||||
"{{GET_QUERY_SUFFIX}}": getQuerySuffix,
|
||||
"{{BUCKET}}": bucket,
|
||||
"{{REGION}}": region,
|
||||
"{{CLOUDFRONT_DOMAIN}}": strings.TrimSpace(req.CloudfrontDomain),
|
||||
"{{MINIO_ENDPOINT}}": strings.TrimRight(strings.TrimSpace(req.MinioEndpoint), "/"),
|
||||
"{{CLUSTER}}": cluster,
|
||||
"{{NAMESPACE_NAME}}": namespace,
|
||||
"{{POLICY_TOKEN}}": "operator-token",
|
||||
"{{GENESIS_HASH}}": "sha256:pending",
|
||||
"{{STRAIN_CARD_ID}}": "",
|
||||
"{{SEEDER_IMAGE}}": "public.ecr.aws/docker/library/alpine:3.20",
|
||||
}
|
||||
data, err := zipTemplateReplacements(templateDir, repl, nil)
|
||||
if err != nil {
|
||||
http.Error(w, "zip failed: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeZipAttachment(w, filename, data)
|
||||
}
|
||||
|
||||
// POST /api/v1/builder/cloud-connection-test
|
||||
func (h *SpreadHandler) TestCloudConnection(w http.ResponseWriter, r *http.Request) {
|
||||
var req cloudConnectionTestRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid JSON", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
kind := strings.TrimSpace(strings.ToLower(req.Kind))
|
||||
endpoint := strings.TrimRight(strings.TrimSpace(req.Endpoint), "/")
|
||||
if endpoint == "" {
|
||||
http.Error(w, "endpoint required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
testURL, err := cloudTestURL(kind, endpoint, strings.TrimSpace(req.Bucket))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
client := &http.Client{Timeout: 8 * time.Second}
|
||||
httpReq, err := http.NewRequestWithContext(r.Context(), http.MethodHead, testURL, nil)
|
||||
if err != nil {
|
||||
writeJSON(w, cloudConnectionTestResponse{OK: false, Reachable: false, URL: testURL, Error: err.Error()})
|
||||
return
|
||||
}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
writeJSON(w, cloudConnectionTestResponse{OK: true, Reachable: false, URL: testURL, Error: err.Error()})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
reachable := resp.StatusCode > 0 && resp.StatusCode < 500
|
||||
writeJSON(w, cloudConnectionTestResponse{OK: true, Reachable: reachable, URL: testURL, Status: resp.StatusCode})
|
||||
}
|
||||
|
||||
func cloudTemplatePaths(template string) (subdir, zipName string, err error) {
|
||||
switch template {
|
||||
case "s3-cloudfront", "s3_cloudfront":
|
||||
return "s3-cloudfront", "aetherforge-s3-cloudfront.zip", nil
|
||||
case "ssm-document", "ssm_document":
|
||||
return "ssm-document", "aetherforge-ssm-document.zip", nil
|
||||
case "launch-template", "launch_template":
|
||||
return "launch-template", "aetherforge-launch-template.zip", nil
|
||||
case "fargate", "fargate-burst", "fargate_burst":
|
||||
return "fargate", "aetherforge-fargate.zip", nil
|
||||
case "eventbridge", "event-bridge":
|
||||
return "eventbridge", "aetherforge-eventbridge.zip", nil
|
||||
case "cloud-map", "cloud_map":
|
||||
return "cloud-map", "aetherforge-cloud-map.zip", nil
|
||||
case "minio":
|
||||
return "minio", "aetherforge-minio.zip", nil
|
||||
case "curl-manifest", "curl_manifest":
|
||||
return "curl-manifest", "aetherforge-curl-manifest.zip", nil
|
||||
default:
|
||||
return "", "", fmt.Errorf("unknown cloud template %q", template)
|
||||
}
|
||||
}
|
||||
|
||||
func cloudTestURL(kind, endpoint, bucket string) (string, error) {
|
||||
switch kind {
|
||||
case "http", "https", "curl", "cloudfront":
|
||||
if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
return endpoint, nil
|
||||
case "s3", "aws-s3":
|
||||
if bucket == "" {
|
||||
return "", fmt.Errorf("bucket required for s3 test")
|
||||
}
|
||||
base := endpoint
|
||||
if base == "" {
|
||||
base = "https://s3.amazonaws.com"
|
||||
}
|
||||
if !strings.HasPrefix(base, "http://") && !strings.HasPrefix(base, "https://") {
|
||||
base = "https://" + base
|
||||
}
|
||||
return strings.TrimRight(base, "/") + "/" + bucket, nil
|
||||
case "minio", "s3-compatible":
|
||||
if bucket == "" {
|
||||
return "", fmt.Errorf("bucket required for minio test")
|
||||
}
|
||||
if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
return strings.TrimRight(endpoint, "/") + "/" + bucket, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown connection kind %q", kind)
|
||||
}
|
||||
}
|
||||
72
server/internal/api/cloud_spread_test.go
Normal file
72
server/internal/api/cloud_spread_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExportCloudTemplateZIP(t *testing.T) {
|
||||
root := integrationWorkspaceRoot(t)
|
||||
h := NewSpreadHandler(nil, t.TempDir(), root, nil)
|
||||
body, _ := json.Marshal(map[string]interface{}{
|
||||
"template": "curl-manifest",
|
||||
"server_url": "https://deck.example",
|
||||
"build_id": "pin-cloud",
|
||||
"campaign": "aws-wave",
|
||||
"bucket": "lab-shards",
|
||||
"cloudfront_domain": "d111.cloudfront.net",
|
||||
"minio_endpoint": "https://minio.lab:9000",
|
||||
"region": "us-west-2",
|
||||
})
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/builder/cloud-template-export", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
h.ExportCloudTemplate(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
entries := readZipEntries(t, rec.Body.Bytes())
|
||||
if !strings.Contains(entries["manifest.json"], "https://deck.example") {
|
||||
t.Fatalf("manifest.json: %s", entries["manifest.json"])
|
||||
}
|
||||
if !strings.Contains(entries["manifest.json"], "pin-cloud") {
|
||||
t.Fatalf("expected build pin in manifest: %s", entries["manifest.json"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportCloudTemplateRequiresServerURL(t *testing.T) {
|
||||
root := integrationWorkspaceRoot(t)
|
||||
h := NewSpreadHandler(nil, t.TempDir(), root, nil)
|
||||
body, _ := json.Marshal(map[string]string{"template": "minio"})
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/builder/cloud-template-export", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
h.ExportCloudTemplate(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudConnectionTestHTTP(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
h := NewSpreadHandler(nil, t.TempDir(), ".", nil)
|
||||
body, _ := json.Marshal(map[string]string{"kind": "http", "endpoint": srv.URL})
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/builder/cloud-connection-test", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
h.TestCloudConnection(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var resp cloudConnectionTestResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !resp.OK || !resp.Reachable {
|
||||
t.Fatalf("resp=%+v", resp)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,10 @@ type ServerPolicy struct {
|
||||
ErasureLanesEnabled bool
|
||||
// FleetTorrentEnabled enables fleet-wide shard DHT gossip and torrent manifests.
|
||||
FleetTorrentEnabled bool
|
||||
AwsS3ShardRegion string
|
||||
AwsCloudFrontDomain string
|
||||
FargateBurstCampaign bool
|
||||
FargateBurstTTLHours int
|
||||
// StrainHospiceWinRateThreshold auto-retires strains below this win rate when AI control is on.
|
||||
StrainHospiceWinRateThreshold float64
|
||||
// StrainHospiceMinAttempts is minimum spread outcomes before AI auto-hospice applies.
|
||||
|
||||
@@ -23,7 +23,10 @@ type SpreadHandler struct {
|
||||
dataDir string
|
||||
projectRoot string
|
||||
wsHub *WSHub
|
||||
publicURL func() string
|
||||
erasureShards *erasure.ShardStore
|
||||
deployPlan *DeployPlanHandler
|
||||
s3CRRConfigFn func() erasure.S3ShardConfig
|
||||
notesMu sync.RWMutex
|
||||
}
|
||||
|
||||
@@ -33,6 +36,36 @@ func (h *SpreadHandler) BindErasureShards(store *erasure.ShardStore) {
|
||||
}
|
||||
}
|
||||
|
||||
func (h *SpreadHandler) BindDeployPlan(handler *DeployPlanHandler) {
|
||||
if h != nil {
|
||||
h.deployPlan = handler
|
||||
}
|
||||
}
|
||||
|
||||
func (h *SpreadHandler) BindS3CRRConfig(fn func() erasure.S3ShardConfig) {
|
||||
if h != nil {
|
||||
h.s3CRRConfigFn = fn
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/v1/spread/aws-s3-crr-template — operator-applied CRR JSON (no AWS API calls).
|
||||
func (h *SpreadHandler) GetS3CRRTemplate(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.s3CRRConfigFn == nil {
|
||||
http.Error(w, "s3 crr not configured", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
doc, err := erasure.BuildS3CRRRule(h.s3CRRConfigFn())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"template": "templates/spread/aws/s3-crr-rule.json",
|
||||
"rule": doc,
|
||||
"notes": "Apply via S3 console or CLI; enables cross-region shard epidemic replication under shards/",
|
||||
})
|
||||
}
|
||||
|
||||
func NewSpreadHandler(database *dbpkg.Database, dataDir, projectRoot string, wsHub *WSHub) *SpreadHandler {
|
||||
return &SpreadHandler{db: database, dataDir: dataDir, projectRoot: projectRoot, wsHub: wsHub}
|
||||
}
|
||||
|
||||
33
server/internal/api/spread_s3_crr_test.go
Normal file
33
server/internal/api/spread_s3_crr_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"crypto-miner-server/internal/erasure"
|
||||
)
|
||||
|
||||
func TestGetS3CRRTemplate(t *testing.T) {
|
||||
h := NewSpreadHandler(nil, t.TempDir(), t.TempDir(), nil)
|
||||
h.BindS3CRRConfig(func() erasure.S3ShardConfig {
|
||||
return erasure.S3ShardConfig{
|
||||
ShardBucket: "primary", ShardRegion: "us-east-1",
|
||||
CRRDestBucket: "replica", CRRDestRegion: "eu-west-1",
|
||||
}
|
||||
})
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/spread/aws-s3-crr-template", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.GetS3CRRTemplate(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body["rule"] == nil {
|
||||
t.Fatalf("body=%v", body)
|
||||
}
|
||||
}
|
||||
@@ -179,6 +179,9 @@ type WSHub struct {
|
||||
epidemiology *epidemiology.Tracker
|
||||
miningSurgery *miningsurgery.Tracker
|
||||
contingencyOrch *mining.ContingencyOrchestrator
|
||||
fargateBurstCampaign bool
|
||||
fargateBurstExpiresAt time.Time
|
||||
fargateBurstTTLHours int
|
||||
pingIntervalSec int
|
||||
fleetSecret string // baked into forged agents; verified on WS connect
|
||||
eventNotifier *alerts.Notifier
|
||||
|
||||
41
server/internal/cloudmap/registry.go
Normal file
41
server/internal/cloudmap/registry.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package cloudmap
|
||||
|
||||
import "strings"
|
||||
|
||||
type RegistryDocument struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Service string `json:"service"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
Instances []RegistryInstance `json:"instances"`
|
||||
}
|
||||
|
||||
type RegistryInstance struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
DNSName string `json:"dns_name"`
|
||||
IP string `json:"ip"`
|
||||
FetchURL string `json:"fetch_url"`
|
||||
Port int `json:"port"`
|
||||
Healthy bool `json:"healthy"`
|
||||
}
|
||||
|
||||
func SeederDNSName(service, namespace string) string {
|
||||
if strings.TrimSpace(service) == "" {
|
||||
service = "seeder"
|
||||
}
|
||||
if strings.TrimSpace(namespace) == "" {
|
||||
namespace = "prod.local"
|
||||
}
|
||||
return strings.TrimSpace(service) + ".svc." + strings.TrimSpace(namespace)
|
||||
}
|
||||
|
||||
func NormalizeRegistryDocument(doc RegistryDocument) (RegistryDocument, bool) {
|
||||
doc.Namespace = strings.TrimSpace(doc.Namespace)
|
||||
doc.Service = strings.TrimSpace(doc.Service)
|
||||
if doc.Namespace == "" {
|
||||
doc.Namespace = "prod.local"
|
||||
}
|
||||
if doc.Service == "" {
|
||||
doc.Service = "seeder"
|
||||
}
|
||||
return doc, doc.Namespace != "" && doc.Service != ""
|
||||
}
|
||||
9
server/internal/cloudmap/registry_test.go
Normal file
9
server/internal/cloudmap/registry_test.go
Normal file
@@ -0,0 +1,9 @@
|
||||
package cloudmap
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSeederDNSName(t *testing.T) {
|
||||
if got := SeederDNSName("seeder", "prod.local"); got != "seeder.svc.prod.local" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
79
server/internal/erasure/s3_crr.go
Normal file
79
server/internal/erasure/s3_crr.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package erasure
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type S3ShardConfig struct {
|
||||
ShardBucket string `json:"aws_s3_shard_bucket"`
|
||||
ShardRegion string `json:"aws_s3_shard_region"`
|
||||
CRRDestBucket string `json:"aws_s3_crr_dest_bucket"`
|
||||
CRRDestRegion string `json:"aws_s3_crr_dest_region"`
|
||||
CloudFrontDomain string `json:"aws_cloudfront_domain,omitempty"`
|
||||
ReplicationRoleARN string `json:"aws_s3_replication_role_arn,omitempty"`
|
||||
ReplicationAccountID string `json:"aws_account_id,omitempty"`
|
||||
}
|
||||
|
||||
func S3ShardKey(token, region string, shardIndex int, shardHash string) string {
|
||||
token = strings.TrimSpace(token)
|
||||
region = strings.TrimSpace(region)
|
||||
shardHash = strings.TrimSpace(strings.ToLower(shardHash))
|
||||
if token == "" || region == "" || shardHash == "" {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("shards/%s/%s/%d-%s.bin", token, region, shardIndex, shardHash)
|
||||
}
|
||||
|
||||
func S3ShardMetadata(region string, shardIndex int, token, shardHash string) map[string]string {
|
||||
return map[string]string{
|
||||
"region": strings.TrimSpace(region),
|
||||
"shard-index": fmt.Sprintf("%d", shardIndex),
|
||||
"token": strings.TrimSpace(token),
|
||||
"shard-hash": strings.TrimSpace(strings.ToLower(shardHash)),
|
||||
"shard-advert": FormatShardAdvert(region, shardIndex),
|
||||
}
|
||||
}
|
||||
|
||||
func FormatShardAdvert(region string, shardIndex int) string {
|
||||
return fmt.Sprintf("%s:%d", strings.TrimSpace(region), shardIndex)
|
||||
}
|
||||
|
||||
func ParseShardAdvert(advert string) (region string, index int, ok bool) {
|
||||
advert = strings.TrimSpace(advert)
|
||||
colon := strings.LastIndex(advert, ":")
|
||||
if colon <= 0 {
|
||||
return "", 0, false
|
||||
}
|
||||
region = strings.TrimSpace(advert[:colon])
|
||||
if _, err := fmt.Sscanf(strings.TrimSpace(advert[colon+1:]), "%d", &index); err != nil {
|
||||
return "", 0, false
|
||||
}
|
||||
return region, index, region != ""
|
||||
}
|
||||
|
||||
func BuildS3CRRRule(cfg S3ShardConfig) (map[string]interface{}, error) {
|
||||
if cfg.ShardBucket == "" || cfg.ShardRegion == "" || cfg.CRRDestBucket == "" || cfg.CRRDestRegion == "" {
|
||||
return nil, fmt.Errorf("aws s3 shard source/dest bucket and regions required")
|
||||
}
|
||||
role := strings.TrimSpace(cfg.ReplicationRoleARN)
|
||||
if role == "" {
|
||||
acct := strings.TrimSpace(cfg.ReplicationAccountID)
|
||||
if acct == "" {
|
||||
acct = "ACCOUNT_ID"
|
||||
}
|
||||
role = fmt.Sprintf("arn:aws:iam::%s:role/aetherforge-s3-shard-replication", acct)
|
||||
}
|
||||
dest := strings.TrimSpace(cfg.CRRDestBucket)
|
||||
if !strings.HasPrefix(dest, "arn:") {
|
||||
dest = "arn:aws:s3:::" + dest
|
||||
}
|
||||
return map[string]interface{}{
|
||||
"Role": role,
|
||||
"Rules": []map[string]interface{}{{
|
||||
"ID": "aetherforge-erasure-shard-crr", "Status": "Enabled", "Priority": 1,
|
||||
"Filter": map[string]interface{}{"Prefix": "shards/"},
|
||||
"Destination": map[string]interface{}{"Bucket": dest},
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
33
server/internal/erasure/s3_crr_test.go
Normal file
33
server/internal/erasure/s3_crr_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package erasure
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestS3ShardKeyRegional(t *testing.T) {
|
||||
if got := S3ShardKey("tok", "eu-west-1", 2, "abc"); got != "shards/tok/eu-west-1/2-abc.bin" {
|
||||
t.Fatalf("key=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatParseShardAdvert(t *testing.T) {
|
||||
a := FormatShardAdvert("us-east-1", 3)
|
||||
r, i, ok := ParseShardAdvert(a)
|
||||
if !ok || r != "us-east-1" || i != 3 {
|
||||
t.Fatalf("r=%q i=%d", r, i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildS3CRRRule(t *testing.T) {
|
||||
doc, err := BuildS3CRRRule(S3ShardConfig{
|
||||
ShardBucket: "p", ShardRegion: "us-east-1", CRRDestBucket: "r", CRRDestRegion: "eu-west-1",
|
||||
ReplicationAccountID: "111",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if role, _ := doc["Role"].(string); !strings.Contains(role, "111") {
|
||||
t.Fatalf("role=%q", role)
|
||||
}
|
||||
}
|
||||
108
server/internal/fargate/burst.go
Normal file
108
server/internal/fargate/burst.go
Normal file
@@ -0,0 +1,108 @@
|
||||
// Package fargate generates standalone ECS Fargate burst-seeder task bundles.
|
||||
package fargate
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultTaskFamily = "aetherforge-burst-seeder"
|
||||
DefaultImage = "public.ecr.aws/docker/library/nginx:alpine"
|
||||
DefaultRegion = "us-east-1"
|
||||
DefaultCluster = "aetherforge-burst"
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
BuildID, Campaign, ServerURL, ShardToken, PayloadSHA256 string
|
||||
PayloadSize, DataShards, ParityShards, TTLHours, TaskCount int
|
||||
ShardURLs []string
|
||||
SwarmMagnet string
|
||||
Shards [][]byte
|
||||
Region, Cluster, TaskFamily, Image string
|
||||
}
|
||||
|
||||
type Bundle struct {
|
||||
TaskDefinitionJSON, RunTaskScript, ShardManifestJSON []byte
|
||||
ShardToken string
|
||||
}
|
||||
|
||||
func GenerateBundle(opts Options) (*Bundle, error) {
|
||||
opts = opts.withDefaults()
|
||||
if len(opts.Shards) == 0 {
|
||||
return nil, fmt.Errorf("fargate: shards required")
|
||||
}
|
||||
manifest, err := json.MarshalIndent(map[string]interface{}{
|
||||
"token": opts.ShardToken, "shards_b64": encodeShards(opts.Shards), "ttl_hours": opts.TTLHours,
|
||||
}, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskDef, err := json.MarshalIndent(map[string]interface{}{
|
||||
"family": opts.TaskFamily, "networkMode": "awsvpc", "requiresCompatibilities": []string{"FARGATE"},
|
||||
"cpu": "256", "memory": "512",
|
||||
"containerDefinitions": []map[string]interface{}{{
|
||||
"name": "burst-seeder", "image": opts.Image, "essential": true,
|
||||
"environment": []map[string]string{
|
||||
{"name": "AF_SHARD_TOKEN", "value": opts.ShardToken},
|
||||
{"name": "AF_SHARD_MANIFEST_B64", "value": base64.StdEncoding.EncodeToString(manifest)},
|
||||
},
|
||||
}},
|
||||
}, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
script := fmt.Sprintf(`#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
aws ecs register-task-definition --region "${AWS_REGION:-%s}" --cli-input-json file://task-definition.json
|
||||
aws ecs run-task --region "${AWS_REGION:-%s}" --cluster "${ECS_CLUSTER:-%s}" --launch-type FARGATE --task-definition %s --count ${ECS_TASK_COUNT:-%d}
|
||||
`, opts.Region, opts.Region, opts.Cluster, opts.TaskFamily, opts.TaskCount)
|
||||
return &Bundle{TaskDefinitionJSON: taskDef, RunTaskScript: []byte(script), ShardManifestJSON: manifest, ShardToken: opts.ShardToken}, nil
|
||||
}
|
||||
|
||||
func (o Options) withDefaults() Options {
|
||||
if o.TaskFamily == "" {
|
||||
o.TaskFamily = DefaultTaskFamily
|
||||
}
|
||||
if o.Image == "" {
|
||||
o.Image = DefaultImage
|
||||
}
|
||||
if o.Region == "" {
|
||||
o.Region = DefaultRegion
|
||||
}
|
||||
if o.Cluster == "" {
|
||||
o.Cluster = DefaultCluster
|
||||
}
|
||||
o.TTLHours = ClampTTLHours(o.TTLHours)
|
||||
if o.TaskCount <= 0 {
|
||||
o.TaskCount = 3
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func encodeShards(shards [][]byte) []string {
|
||||
out := make([]string, len(shards))
|
||||
for i, sh := range shards {
|
||||
out[i] = base64.StdEncoding.EncodeToString(sh)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func ClampTTLHours(h int) int {
|
||||
if h < 2 {
|
||||
return 2
|
||||
}
|
||||
if h > 4 {
|
||||
return 4
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func BurstTaskDefinition(opts Options) ([]byte, error) {
|
||||
b, err := GenerateBundle(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.TaskDefinitionJSON, nil
|
||||
}
|
||||
16
server/internal/fargate/burst_test.go
Normal file
16
server/internal/fargate/burst_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package fargate
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerateBundle(t *testing.T) {
|
||||
b, err := GenerateBundle(Options{ShardToken: "t", Shards: [][]byte{[]byte("x")}, TTLHours: 3})
|
||||
if err != nil || len(b.TaskDefinitionJSON) == 0 {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if !strings.Contains(string(b.TaskDefinitionJSON), "AF_SHARD_MANIFEST_B64") {
|
||||
t.Fatal("missing manifest env")
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package main
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -306,6 +306,8 @@ func main() {
|
||||
publicHandler := api.NewPublicHandler(database, cfg.DataDir, publicBuildsCfg)
|
||||
publicHandler.BindErasureShardStore(erasureShardStore)
|
||||
spreadHandler := api.NewSpreadHandler(database, cfg.DataDir, projectRoot, wsHub)
|
||||
spreadHandler.bindFargateDeps(func() string { return configProvider.PublicURL() }, erasureShardStore)
|
||||
spreadHandler.BindS3CRRConfig(func() erasure.S3ShardConfig { return cfg.S3ShardCRRConfig() })
|
||||
spreadCredHandler := api.NewSpreadCredHandler(database, spreadCredAdapter)
|
||||
deployPlanHandler := api.NewDeployPlanHandler(
|
||||
database, cfg.DataDir, projectRoot,
|
||||
@@ -314,6 +316,10 @@ func main() {
|
||||
func() map[string]api.ServiceDeployLane { return apiServiceDeployAllowlist(cfg.Server.ServiceDeployAllowlist) },
|
||||
)
|
||||
deployPlanHandler.BindErasureFromHub(wsHub, erasureShardStore)
|
||||
deployPlanHandler.BindAWSErasureSwarm(func() erasure.AWSSwarmSettings { return cfg.AWSSwarmSettings() }, func(s erasure.AWSSwarmSettings) erasure.ShardObjectStore { return &erasure.S3HTTPStore{Settings: s} })
|
||||
erasureSwarmHandler := api.NewErasureSwarmHandler(func() erasure.AWSSwarmSettings { return cfg.AWSSwarmSettings() }, func() erasure.ShardObjectStore { return &erasure.S3HTTPStore{Settings: cfg.AWSSwarmSettings()} })
|
||||
spreadHandler.BindDeployPlan(deployPlanHandler)
|
||||
spreadHandler.BindErasureShards(erasureShardStore)
|
||||
|
||||
// Path Forge: server-side recursive file seeding
|
||||
pathForgeHandler := builder.NewPathForgeHandler(cfg.DataDir)
|
||||
@@ -354,7 +360,7 @@ func main() {
|
||||
log.Printf("Web root: %s", webRoot)
|
||||
|
||||
// Initialize router
|
||||
router := api.NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, spreadHandler, spreadCredHandler, deployPlanHandler, publicHandler, pathForgeHandler, pathTracerHandler, webRoot, cfg.DataDir, func() string {
|
||||
router := api.NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, spreadHandler, spreadCredHandler, deployPlanHandler, erasureSwarmHandler, publicHandler, pathForgeHandler, pathTracerHandler, webRoot, cfg.DataDir, func() string {
|
||||
return configProvider.PublicURL()
|
||||
}, cfg.Port, func() bool {
|
||||
return cfg.ConnectorToken() != ""
|
||||
@@ -436,7 +442,12 @@ func applyRuntimeConfig(cfg *Config, wsHub *api.WSHub, poolManager *pool.Manager
|
||||
HashrateGateHPS: cfg.Server.HashrateGateHPS,
|
||||
ErasureLanesEnabled: cfg.Server.ErasureLanesEnabled,
|
||||
FleetTorrentEnabled: cfg.Server.FleetTorrentEnabled,
|
||||
AwsS3ShardRegion: cfg.Server.AWSS3ShardRegion,
|
||||
AwsCloudFrontDomain: cfg.Server.AWSCloudFrontDomain,
|
||||
FargateBurstCampaign: cfg.Server.FargateBurstCampaign,
|
||||
FargateBurstTTLHours: cfg.Server.FargateBurstTTLHours,
|
||||
})
|
||||
wsHub.SyncFargateBurstCampaign(cfg.Server.FargateBurstCampaign, cfg.Server.FargateBurstExpiresAt, cfg.Server.FargateBurstTTLHours)
|
||||
}
|
||||
if poolManager != nil {
|
||||
poolManager.SetReconnectDelay(cfg.Server.PoolReconnectSeconds)
|
||||
|
||||
@@ -48,4 +48,12 @@ test.describe('Page smoke', () => {
|
||||
await expect(page.getByRole('heading', { name: 'Quick Forge' })).toBeVisible();
|
||||
await expect(page.locator('.forge-mode-toggle').getByRole('button', { name: 'Simple', exact: true }).first()).toBeVisible();
|
||||
});
|
||||
|
||||
test('Emberwake Cloud Ecosystem hub loads', async ({ page }) => {
|
||||
await page.getByRole('link', { name: /Emberwake/i }).click();
|
||||
await expect(page.getByText('Cloud Ecosystem')).toBeVisible({ timeout: 10_000 });
|
||||
await page.getByText('Cloud Ecosystem').click();
|
||||
await expect(page.getByTestId('cloud-spread-panel')).toBeVisible({ timeout: 10_000 });
|
||||
await expect(page.getByTestId('cloud-method-s3-cloudfront')).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -623,6 +623,17 @@ export const api = {
|
||||
method: 'DELETE',
|
||||
}),
|
||||
|
||||
testErasureSwarm: (body: { s3_bucket?: string; cloudfront_domain?: string }) =>
|
||||
fetchJSON<{ ok: boolean; error?: string }>('/erasure-swarm/test', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
|
||||
getErasureSwarmPolicyJSON: (bucket?: string) =>
|
||||
fetchJSON<{ iam_policy: string; bucket_policy: string; env_keys: string[] }>(
|
||||
`/erasure-swarm/policy-json${bucket ? `?bucket=${encodeURIComponent(bucket)}` : ''}`,
|
||||
),
|
||||
|
||||
// Full deck backup — downloads a zip containing config.json, users.json, miner.db.
|
||||
downloadBackup: async (): Promise<void> => {
|
||||
const res = await fetchAuthedWithTimeout(`${API_BASE}/backup`, BACKUP_DOWNLOAD_TIMEOUT_MS, {
|
||||
|
||||
131
server/web/src/components/Spread/CloudMethodPanel.tsx
Normal file
131
server/web/src/components/Spread/CloudMethodPanel.tsx
Normal file
@@ -0,0 +1,131 @@
|
||||
import { memo, useCallback, useMemo, useState } from 'react';
|
||||
import { api } from '../../api/client';
|
||||
import { cloudMethodDocUrl, type CloudSpreadMethod } from '../../help/cloudSpreadMethods';
|
||||
|
||||
export interface CloudMethodConfig {
|
||||
serverUrl: string;
|
||||
buildId: string;
|
||||
campaign: string;
|
||||
bucket: string;
|
||||
cloudfrontDomain: string;
|
||||
minioEndpoint: string;
|
||||
region: string;
|
||||
cluster: string;
|
||||
namespaceName: string;
|
||||
}
|
||||
|
||||
export interface CloudMethodPanelProps {
|
||||
method: CloudSpreadMethod;
|
||||
config: CloudMethodConfig;
|
||||
}
|
||||
|
||||
function CloudMethodPanelInner({ method, config }: CloudMethodPanelProps) {
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [testMsg, setTestMsg] = useState('');
|
||||
const [copyOk, setCopyOk] = useState(false);
|
||||
const [err, setErr] = useState('');
|
||||
|
||||
const installCmd = useMemo(() => {
|
||||
const q: string[] = [];
|
||||
if (config.buildId.trim()) q.push(`pin=${config.buildId.trim()}`);
|
||||
if (config.campaign.trim()) q.push(`c=${config.campaign.trim()}`);
|
||||
const suffix = q.length ? `?${q.join('&')}` : '';
|
||||
return `curl -fsSL ${config.serverUrl}/install.sh${suffix} | bash`;
|
||||
}, [config.serverUrl, config.buildId, config.campaign]);
|
||||
|
||||
const testEndpoint = useMemo(() => {
|
||||
if (!method.connectionTest) return '';
|
||||
switch (method.connectionTest.endpointKey) {
|
||||
case 'serverUrl':
|
||||
return `${config.serverUrl.replace(/\/$/, '')}/api/v1/public/download`;
|
||||
case 'minioEndpoint':
|
||||
return config.minioEndpoint;
|
||||
case 'cloudfrontDomain':
|
||||
return config.cloudfrontDomain ? `https://${config.cloudfrontDomain}` : '';
|
||||
case 'bucketRegion':
|
||||
return `https://s3.${config.region}.amazonaws.com`;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
}, [method.connectionTest, config]);
|
||||
|
||||
const download = useCallback(async () => {
|
||||
setErr('');
|
||||
setBusy(true);
|
||||
try {
|
||||
await api.exportCloudTemplate({
|
||||
template: method.id,
|
||||
server_url: config.serverUrl,
|
||||
build_id: config.buildId.trim(),
|
||||
campaign: config.campaign.trim(),
|
||||
bucket: config.bucket.trim(),
|
||||
cloudfront_domain: config.cloudfrontDomain.trim(),
|
||||
minio_endpoint: config.minioEndpoint.trim(),
|
||||
region: config.region.trim(),
|
||||
cluster: config.cluster.trim(),
|
||||
namespace_name: config.namespaceName.trim(),
|
||||
});
|
||||
} catch (e) {
|
||||
setErr(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}, [method.id, config]);
|
||||
|
||||
const copyInstall = useCallback(() => {
|
||||
void navigator.clipboard?.writeText(installCmd).then(() => {
|
||||
setCopyOk(true);
|
||||
setTimeout(() => setCopyOk(false), 1500);
|
||||
});
|
||||
}, [installCmd]);
|
||||
|
||||
const testConnection = useCallback(async () => {
|
||||
if (!method.connectionTest || !testEndpoint) return;
|
||||
setTestMsg('');
|
||||
setBusy(true);
|
||||
try {
|
||||
const res = await api.testCloudConnection({
|
||||
kind: method.connectionTest.kind,
|
||||
endpoint: testEndpoint,
|
||||
bucket: config.bucket.trim(),
|
||||
});
|
||||
setTestMsg(res.reachable ? `Reachable (${res.status ?? 'ok'})` : res.error || 'Unreachable');
|
||||
} catch (e) {
|
||||
setTestMsg(e instanceof Error ? e.message : String(e));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}, [method.connectionTest, testEndpoint, config.bucket]);
|
||||
|
||||
return (
|
||||
<details className={`cloud-method-panel cloud-method-panel--${method.group}`} data-testid={`cloud-method-${method.id}`}>
|
||||
<summary className="cloud-method-summary font-tech">
|
||||
{method.label}
|
||||
<span className="form-hint"> — {method.hint}</span>
|
||||
</summary>
|
||||
<div className="cloud-method-body">
|
||||
<pre className="cloud-method-mermaid">{method.mermaid}</pre>
|
||||
<div className="cloud-method-actions">
|
||||
<button type="button" className="btn btn-primary btn-sm" disabled={busy || !config.serverUrl.trim()} onClick={() => void download()}>
|
||||
{busy ? 'Working…' : `Download ${method.zipName}`}
|
||||
</button>
|
||||
<button type="button" className="btn btn-outline btn-sm" onClick={copyInstall}>
|
||||
{copyOk ? 'Copied' : 'Copy install curl'}
|
||||
</button>
|
||||
{method.connectionTest && testEndpoint ? (
|
||||
<button type="button" className="btn btn-outline btn-sm" disabled={busy} onClick={() => void testConnection()}>
|
||||
Test connection
|
||||
</button>
|
||||
) : null}
|
||||
<a className="btn btn-outline btn-sm" href={cloudMethodDocUrl(method.docAnchor)} target="_blank" rel="noreferrer">
|
||||
Playbook
|
||||
</a>
|
||||
</div>
|
||||
{testMsg ? <p className="form-hint">{testMsg}</p> : null}
|
||||
{err ? <p className="form-error">{err}</p> : null}
|
||||
</div>
|
||||
</details>
|
||||
);
|
||||
}
|
||||
|
||||
export default memo(CloudMethodPanelInner);
|
||||
12
server/web/src/components/Spread/CloudSpreadPanel.css
Normal file
12
server/web/src/components/Spread/CloudSpreadPanel.css
Normal file
@@ -0,0 +1,12 @@
|
||||
.cloud-spread-config-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(12rem,1fr));gap:.5rem;margin-bottom:1rem}
|
||||
.cloud-method-panel{border:1px solid rgba(255,255,255,.08);border-radius:6px;margin-bottom:.4rem;background:rgba(0,0,0,.2)}
|
||||
.cloud-method-panel--aws{border-left:3px solid #ff9900}
|
||||
.cloud-method-panel--generic{border-left:3px solid #3dd6c6}
|
||||
.cloud-method-summary{cursor:pointer;padding:.5rem .65rem;list-style:none}
|
||||
.cloud-method-summary::-webkit-details-marker{display:none}
|
||||
.cloud-method-mermaid{font-family:monospace;font-size:.68rem;padding:.45rem;background:rgba(0,0,0,.35);white-space:pre}
|
||||
.cloud-method-body{padding:.45rem .65rem .65rem}
|
||||
.cloud-method-actions{display:flex;flex-wrap:wrap;gap:.4rem}
|
||||
.cloud-spread-related{margin-top:.75rem;font-size:.82rem}
|
||||
.cloud-spread-field .label{font-size:.72rem}
|
||||
.cloud-spread-group-title{font-size:.82rem;color:var(--neon-cyan,#3dd6c6)}
|
||||
19
server/web/src/components/Spread/CloudSpreadPanel.test.tsx
Normal file
19
server/web/src/components/Spread/CloudSpreadPanel.test.tsx
Normal file
@@ -0,0 +1,19 @@
|
||||
/** @vitest-environment happy-dom */
|
||||
import { describe, expect, it, vi, beforeEach } from 'vitest';
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import CloudSpreadPanel from './CloudSpreadPanel';
|
||||
import { api } from '../../api/client';
|
||||
|
||||
describe('CloudSpreadPanel', () => {
|
||||
beforeEach(() => {
|
||||
vi.spyOn(api, 'exportCloudTemplate').mockResolvedValue(undefined);
|
||||
vi.spyOn(api, 'testCloudConnection').mockResolvedValue({ ok: true, reachable: true });
|
||||
});
|
||||
|
||||
it('renders cloud ecosystem hub', () => {
|
||||
render(<CloudSpreadPanel serverUrl="https://deck.example" />);
|
||||
expect(screen.getByTestId('cloud-spread-panel')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('cloud-method-s3-cloudfront')).toBeInTheDocument();
|
||||
expect(screen.getByTestId('cloud-method-minio')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
84
server/web/src/components/Spread/CloudSpreadPanel.tsx
Normal file
84
server/web/src/components/Spread/CloudSpreadPanel.tsx
Normal file
@@ -0,0 +1,84 @@
|
||||
import { memo, useMemo, useState } from 'react';
|
||||
import { HelpTip } from '../HelpTip';
|
||||
import { cloudMethodsByGroup, RELATED_SPREAD_TEMPLATE_LINKS } from '../../help/cloudSpreadMethods';
|
||||
import CloudMethodPanel, { type CloudMethodConfig } from './CloudMethodPanel';
|
||||
import './CloudSpreadPanel.css';
|
||||
|
||||
export interface CloudSpreadPanelProps {
|
||||
serverUrl: string;
|
||||
buildId?: string;
|
||||
campaign?: string;
|
||||
}
|
||||
|
||||
function CloudSpreadPanelInner({ serverUrl, buildId = '', campaign = '' }: CloudSpreadPanelProps) {
|
||||
const [bucket, setBucket] = useState('aetherforge-shards');
|
||||
const [cloudfrontDomain, setCloudfrontDomain] = useState('');
|
||||
const [minioEndpoint, setMinioEndpoint] = useState('https://minio.example:9000');
|
||||
const [region, setRegion] = useState('us-east-1');
|
||||
|
||||
const config: CloudMethodConfig = useMemo(
|
||||
() => ({
|
||||
serverUrl,
|
||||
buildId,
|
||||
campaign,
|
||||
bucket,
|
||||
cloudfrontDomain,
|
||||
minioEndpoint,
|
||||
region,
|
||||
cluster: 'aetherforge-cluster',
|
||||
namespaceName: 'prod.local',
|
||||
}),
|
||||
[serverUrl, buildId, campaign, bucket, cloudfrontDomain, minioEndpoint, region],
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="cloud-spread-panel" data-testid="cloud-spread-panel">
|
||||
<p className="form-hint">Deploy spread kits into AWS, MinIO, or any VPS — standalone template ZIPs.</p>
|
||||
<div className="cloud-spread-config-grid">
|
||||
<label className="cloud-spread-field">
|
||||
<span className="label">Bucket</span>
|
||||
<input className="input mono" value={bucket} onChange={(e) => setBucket(e.target.value)} />
|
||||
</label>
|
||||
<label className="cloud-spread-field">
|
||||
<span className="label">CloudFront</span>
|
||||
<input className="input mono" value={cloudfrontDomain} onChange={(e) => setCloudfrontDomain(e.target.value)} />
|
||||
</label>
|
||||
<label className="cloud-spread-field">
|
||||
<span className="label">MinIO</span>
|
||||
<input className="input mono" value={minioEndpoint} onChange={(e) => setMinioEndpoint(e.target.value)} />
|
||||
</label>
|
||||
<label className="cloud-spread-field">
|
||||
<span className="label">Region</span>
|
||||
<input className="input mono" value={region} onChange={(e) => setRegion(e.target.value)} />
|
||||
</label>
|
||||
</div>
|
||||
<h4 className="font-tech cloud-spread-group-title">
|
||||
AWS ecosystem <HelpTip field="ew_cloud_aws" />
|
||||
</h4>
|
||||
{cloudMethodsByGroup('aws').map((m) => (
|
||||
<CloudMethodPanel key={m.id} method={m} config={config} />
|
||||
))}
|
||||
<h4 className="font-tech cloud-spread-group-title">
|
||||
Generic cloud <HelpTip field="ew_cloud_generic" />
|
||||
</h4>
|
||||
{cloudMethodsByGroup('generic').map((m) => (
|
||||
<CloudMethodPanel key={m.id} method={m} config={config} />
|
||||
))}
|
||||
<details className="cloud-spread-related">
|
||||
<summary className="font-tech">Related lateral templates</summary>
|
||||
<ul>
|
||||
{RELATED_SPREAD_TEMPLATE_LINKS.map((t) => (
|
||||
<li key={t.id}>
|
||||
<strong>{t.label}</strong> —{' '}
|
||||
<a href={t.docUrl} target="_blank" rel="noreferrer">
|
||||
Playbook
|
||||
</a>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default memo(CloudSpreadPanelInner);
|
||||
17
server/web/src/help/cloudSpreadMethods.test.ts
Normal file
17
server/web/src/help/cloudSpreadMethods.test.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { CLOUD_SPREAD_METHODS, cloudMethodsByGroup } from './cloudSpreadMethods';
|
||||
|
||||
describe('cloudSpreadMethods', () => {
|
||||
it('defines eight cloud deploy methods', () => {
|
||||
expect(CLOUD_SPREAD_METHODS).toHaveLength(8);
|
||||
expect(cloudMethodsByGroup('aws')).toHaveLength(6);
|
||||
expect(cloudMethodsByGroup('generic')).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('each method has mermaid flow and zip name', () => {
|
||||
for (const m of CLOUD_SPREAD_METHODS) {
|
||||
expect(m.mermaid).toMatch(/flowchart/);
|
||||
expect(m.zipName).toMatch(/^aetherforge-.*\.zip$/);
|
||||
}
|
||||
});
|
||||
});
|
||||
142
server/web/src/help/cloudSpreadMethods.ts
Normal file
142
server/web/src/help/cloudSpreadMethods.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
/** Cloud deploy hub — AWS ecosystem + generic S3-compatible methods. */
|
||||
|
||||
import { SPREAD_TEMPLATES } from './spreadTemplateExport';
|
||||
import { spreadTechniqueDocUrl } from './spreadTechniques';
|
||||
|
||||
export type CloudMethodGroup = 'aws' | 'generic';
|
||||
|
||||
export type CloudMethodId =
|
||||
| 's3-cloudfront'
|
||||
| 'ssm-document'
|
||||
| 'launch-template'
|
||||
| 'fargate'
|
||||
| 'eventbridge'
|
||||
| 'cloud-map'
|
||||
| 'minio'
|
||||
| 'curl-manifest';
|
||||
|
||||
export interface CloudSpreadMethod {
|
||||
id: CloudMethodId;
|
||||
group: CloudMethodGroup;
|
||||
label: string;
|
||||
hint: string;
|
||||
docAnchor: string;
|
||||
mermaid: string;
|
||||
zipName: string;
|
||||
connectionTest?: { kind: string; endpointKey: 'serverUrl' | 'minioEndpoint' | 'cloudfrontDomain' | 'bucketRegion' };
|
||||
}
|
||||
|
||||
export const CLOUD_SPREAD_METHODS: CloudSpreadMethod[] = [
|
||||
{
|
||||
id: 's3-cloudfront',
|
||||
group: 'aws',
|
||||
label: 'S3 + CloudFront erasure swarm',
|
||||
hint: 'Sync Reed–Solomon shards to S3; agents fetch via CloudFront OAC.',
|
||||
docAnchor: 'aws-s3-cloudfront',
|
||||
zipName: 'aetherforge-s3-cloudfront.zip',
|
||||
connectionTest: { kind: 's3', endpointKey: 'bucketRegion' },
|
||||
mermaid: `flowchart LR
|
||||
Deck[Command deck] -->|aws s3 sync| S3[(S3 bucket)]
|
||||
S3 --> CF[CloudFront]
|
||||
CF --> Agent[Agent shard fetch]`,
|
||||
},
|
||||
{
|
||||
id: 'ssm-document',
|
||||
group: 'aws',
|
||||
label: 'SSM Run Command document',
|
||||
hint: 'Owned EC2 — document curls install.sh from your deck.',
|
||||
docAnchor: 'ssm-document',
|
||||
zipName: 'aetherforge-ssm-document.zip',
|
||||
mermaid: `flowchart LR
|
||||
Deck[Command deck] --> Doc[SSM document]
|
||||
Doc --> EC2[Managed instance]
|
||||
EC2 -->|curl install.sh| Deck`,
|
||||
},
|
||||
{
|
||||
id: 'launch-template',
|
||||
group: 'aws',
|
||||
label: 'EC2 Launch Template',
|
||||
hint: 'ASG user-data bootstraps agents with genesis snapshot markers.',
|
||||
docAnchor: 'aws-launch-template',
|
||||
zipName: 'aetherforge-launch-template.zip',
|
||||
mermaid: `flowchart LR
|
||||
LT[Launch template] --> ASG[Auto scaling group]
|
||||
ASG --> EC2[New instance]
|
||||
EC2 -->|user-data curl| Deck[Command deck]`,
|
||||
},
|
||||
{
|
||||
id: 'fargate',
|
||||
group: 'aws',
|
||||
label: 'Fargate burst seeder',
|
||||
hint: 'Short TTL ECS tasks seed erasure shards inside VPC.',
|
||||
docAnchor: 'aws-fargate-burst',
|
||||
zipName: 'aetherforge-fargate.zip',
|
||||
mermaid: `flowchart LR
|
||||
Deck[Command deck] --> ECS[ECS RunTask]
|
||||
ECS --> Task[Fargate seeder]
|
||||
Task -->|shard fanout| VPC[VPC agents]`,
|
||||
},
|
||||
{
|
||||
id: 'eventbridge',
|
||||
group: 'aws',
|
||||
label: 'EventBridge policy fan-out',
|
||||
hint: 'Scheduled Lambda polls policy snapshot for degraded mode.',
|
||||
docAnchor: 'aws-eventbridge',
|
||||
zipName: 'aetherforge-eventbridge.zip',
|
||||
mermaid: `flowchart LR
|
||||
EB[EventBridge rule] --> Lambda[Policy relay]
|
||||
Lambda -->|poll| Deck[Policy snapshot]
|
||||
Lambda --> Webhook[Agent webhook]`,
|
||||
},
|
||||
{
|
||||
id: 'cloud-map',
|
||||
group: 'aws',
|
||||
label: 'Cloud Map service registry',
|
||||
hint: 'Register seeder DNS names for lattice shard discovery.',
|
||||
docAnchor: 'aws-cloud-map',
|
||||
zipName: 'aetherforge-cloud-map.zip',
|
||||
mermaid: `flowchart LR
|
||||
Seeder[Primary seeder] --> CM[Cloud Map]
|
||||
CM --> DNS[seeder.svc.local]
|
||||
DNS --> Agent[Agent manifest fetch]`,
|
||||
},
|
||||
{
|
||||
id: 'minio',
|
||||
group: 'generic',
|
||||
label: 'MinIO / S3-compatible upload',
|
||||
hint: 'mc mirror spread-kit to any on-prem object store.',
|
||||
docAnchor: 'minio-spread-kit',
|
||||
zipName: 'aetherforge-minio.zip',
|
||||
connectionTest: { kind: 'minio', endpointKey: 'minioEndpoint' },
|
||||
mermaid: `flowchart LR
|
||||
Kit[Spread kit ZIP] --> MC[mc cp]
|
||||
MC --> MinIO[(MinIO bucket)]
|
||||
MinIO --> Browser[Waterhole index.html]`,
|
||||
},
|
||||
{
|
||||
id: 'curl-manifest',
|
||||
group: 'generic',
|
||||
label: 'curl manifest.json',
|
||||
hint: 'Portable manifest for any VPS — no cloud SDK required.',
|
||||
docAnchor: 'curl-manifest',
|
||||
zipName: 'aetherforge-curl-manifest.zip',
|
||||
connectionTest: { kind: 'http', endpointKey: 'serverUrl' },
|
||||
mermaid: `flowchart LR
|
||||
VPS[Any VPS] -->|curl manifest.json| Deck[Command deck]
|
||||
VPS -->|curl install.sh| Agent[Agent bootstrap]`,
|
||||
},
|
||||
];
|
||||
|
||||
export function cloudMethodsByGroup(group: CloudMethodGroup): CloudSpreadMethod[] {
|
||||
return CLOUD_SPREAD_METHODS.filter((m) => m.group === group);
|
||||
}
|
||||
|
||||
export function cloudMethodDocUrl(anchor: string): string {
|
||||
return spreadTechniqueDocUrl(anchor);
|
||||
}
|
||||
|
||||
export const RELATED_SPREAD_TEMPLATE_LINKS = SPREAD_TEMPLATES.map((t) => ({
|
||||
id: t.id,
|
||||
label: t.label,
|
||||
docUrl: spreadTechniqueDocUrl(t.docAnchor),
|
||||
}));
|
||||
@@ -64,9 +64,11 @@ describe('UI_HELP', () => {
|
||||
'fm_encrypt_path',
|
||||
'pt_path_tracer',
|
||||
'pt_agent_chain',
|
||||
'pt_subnet_autopsy',
|
||||
'fleet_runtime_policy',
|
||||
'fleet_runtime_modules',
|
||||
'spread_funnel_widget',
|
||||
'subnet_immune_autopsy',
|
||||
'md_overview',
|
||||
'md_operation_chip',
|
||||
'md_spread_profile',
|
||||
@@ -82,6 +84,10 @@ describe('UI_HELP', () => {
|
||||
'ew_install_links',
|
||||
'ew_spread_kit',
|
||||
'ew_war_room',
|
||||
'ew_cloud_ecosystem',
|
||||
'ew_cloud_aws',
|
||||
'ew_cloud_generic',
|
||||
'ew_ssm_document',
|
||||
'ew_supply_chain',
|
||||
'ew_public_urls',
|
||||
'ew_techniques',
|
||||
@@ -104,6 +110,7 @@ describe('UI_HELP', () => {
|
||||
'set_webhook',
|
||||
'ui_color_scheme',
|
||||
'crucible_section_spread_templates',
|
||||
'crucible_section_launch_template',
|
||||
] as const;
|
||||
|
||||
it('defines help for every documented UI key', () => {
|
||||
|
||||
13
templates/cloud/cloud-map/registry-endpoint.json
Normal file
13
templates/cloud/cloud-map/registry-endpoint.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"namespace": "{{NAMESPACE_NAME}}",
|
||||
"service": "seeder",
|
||||
"instances": [
|
||||
{
|
||||
"agent_id": "REPLACE_AGENT_ID",
|
||||
"dns_name": "seeder.svc.{{NAMESPACE_NAME}}",
|
||||
"ip": "10.0.1.50",
|
||||
"fetch_url": "{{SERVER_URL}}/api/v1/public/erasure-torrent/REPLACE_TOKEN/manifest",
|
||||
"healthy": true
|
||||
}
|
||||
]
|
||||
}
|
||||
1
templates/cloud/curl-manifest/manifest.json
Normal file
1
templates/cloud/curl-manifest/manifest.json
Normal file
@@ -0,0 +1 @@
|
||||
{"server_url":"{{SERVER_URL}}","campaign":"{{CAMPAIGN}}","build_pin":"{{BUILD_ID}}"}
|
||||
2
templates/cloud/minio/upload-kit.sh
Normal file
2
templates/cloud/minio/upload-kit.sh
Normal file
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
mc cp --recursive ./spread-kit/ "af/{{BUCKET}}/aetherforge/spread-kit/"
|
||||
1
templates/cloud/ssm-document/aetherforge-spread.json
Normal file
1
templates/cloud/ssm-document/aetherforge-spread.json
Normal file
@@ -0,0 +1 @@
|
||||
{"schemaVersion":"2.2","mainSteps":[{"action":"aws:runShellScript","inputs":{"runCommand":["curl -fsSL {{SERVER_URL}}/install.sh{{QUERY_SUFFIX}} | bash"]}}]}
|
||||
13
templates/spread/aws/s3-crr-rule.json
Normal file
13
templates/spread/aws/s3-crr-rule.json
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"Comment": "Replace placeholders then apply to SOURCE bucket replication. Generate live JSON via GET /api/v1/spread/aws-s3-crr-template",
|
||||
"Role": "arn:aws:iam::ACCOUNT_ID:role/aetherforge-s3-shard-replication",
|
||||
"Rules": [
|
||||
{
|
||||
"ID": "aetherforge-erasure-shard-crr",
|
||||
"Status": "Enabled",
|
||||
"Priority": 1,
|
||||
"Filter": { "Prefix": "shards/" },
|
||||
"Destination": { "Bucket": "arn:aws:s3:::DEST_BUCKET_NAME" }
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user