Add Emberwake Cloud Ecosystem deploy hub with AWS and generic spread templates.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Unified expandable panels with mermaid flows, ZIP export, connection tests, and Playwright smoke coverage.
This commit is contained in:
198
server/internal/api/cloud_spread.go
Normal file
198
server/internal/api/cloud_spread.go
Normal file
@@ -0,0 +1,198 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type cloudTemplateExportRequest struct {
|
||||
Template string `json:"template"`
|
||||
ServerURL string `json:"server_url"`
|
||||
BuildID string `json:"build_id"`
|
||||
Campaign string `json:"campaign"`
|
||||
Bucket string `json:"bucket"`
|
||||
CloudfrontDomain string `json:"cloudfront_domain"`
|
||||
MinioEndpoint string `json:"minio_endpoint"`
|
||||
Region string `json:"region"`
|
||||
Cluster string `json:"cluster"`
|
||||
NamespaceName string `json:"namespace_name"`
|
||||
}
|
||||
|
||||
type cloudConnectionTestRequest struct {
|
||||
Kind string `json:"kind"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Bucket string `json:"bucket"`
|
||||
}
|
||||
|
||||
type cloudConnectionTestResponse struct {
|
||||
OK bool `json:"ok"`
|
||||
Reachable bool `json:"reachable"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Status int `json:"status,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// POST /api/v1/builder/cloud-template-export
|
||||
func (h *SpreadHandler) ExportCloudTemplate(w http.ResponseWriter, r *http.Request) {
|
||||
var req cloudTemplateExportRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid JSON", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
req.Template = strings.TrimSpace(strings.ToLower(req.Template))
|
||||
req.ServerURL = strings.TrimRight(strings.TrimSpace(req.ServerURL), "/")
|
||||
req.BuildID = strings.TrimSpace(req.BuildID)
|
||||
req.Campaign = strings.TrimSpace(req.Campaign)
|
||||
if req.ServerURL == "" {
|
||||
http.Error(w, "server_url required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if req.Template == "" {
|
||||
http.Error(w, "template required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
subdir, filename, err := cloudTemplatePaths(req.Template)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
templateDir := filepath.Join(h.projectRoot, "templates", "cloud", subdir)
|
||||
if _, err := os.Stat(templateDir); err != nil {
|
||||
http.Error(w, "cloud template not found: "+subdir, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
querySuffix, getQuerySuffix := buildQuerySuffix(req.BuildID, req.Campaign)
|
||||
bucket := strings.TrimSpace(req.Bucket)
|
||||
if bucket == "" {
|
||||
bucket = "aetherforge-shards"
|
||||
}
|
||||
region := strings.TrimSpace(req.Region)
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
cluster := strings.TrimSpace(req.Cluster)
|
||||
if cluster == "" {
|
||||
cluster = "aetherforge-cluster"
|
||||
}
|
||||
namespace := strings.TrimSpace(req.NamespaceName)
|
||||
if namespace == "" {
|
||||
namespace = "prod.local"
|
||||
}
|
||||
repl := map[string]string{
|
||||
"{{SERVER_URL}}": req.ServerURL,
|
||||
"{{BUILD_ID}}": req.BuildID,
|
||||
"{{CAMPAIGN}}": req.Campaign,
|
||||
"{{QUERY_SUFFIX}}": querySuffix,
|
||||
"{{GET_QUERY_SUFFIX}}": getQuerySuffix,
|
||||
"{{BUCKET}}": bucket,
|
||||
"{{REGION}}": region,
|
||||
"{{CLOUDFRONT_DOMAIN}}": strings.TrimSpace(req.CloudfrontDomain),
|
||||
"{{MINIO_ENDPOINT}}": strings.TrimRight(strings.TrimSpace(req.MinioEndpoint), "/"),
|
||||
"{{CLUSTER}}": cluster,
|
||||
"{{NAMESPACE_NAME}}": namespace,
|
||||
"{{POLICY_TOKEN}}": "operator-token",
|
||||
"{{GENESIS_HASH}}": "sha256:pending",
|
||||
"{{STRAIN_CARD_ID}}": "",
|
||||
"{{SEEDER_IMAGE}}": "public.ecr.aws/docker/library/alpine:3.20",
|
||||
}
|
||||
data, err := zipTemplateReplacements(templateDir, repl, nil)
|
||||
if err != nil {
|
||||
http.Error(w, "zip failed: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
writeZipAttachment(w, filename, data)
|
||||
}
|
||||
|
||||
// POST /api/v1/builder/cloud-connection-test
|
||||
func (h *SpreadHandler) TestCloudConnection(w http.ResponseWriter, r *http.Request) {
|
||||
var req cloudConnectionTestRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid JSON", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
kind := strings.TrimSpace(strings.ToLower(req.Kind))
|
||||
endpoint := strings.TrimRight(strings.TrimSpace(req.Endpoint), "/")
|
||||
if endpoint == "" {
|
||||
http.Error(w, "endpoint required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
testURL, err := cloudTestURL(kind, endpoint, strings.TrimSpace(req.Bucket))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
client := &http.Client{Timeout: 8 * time.Second}
|
||||
httpReq, err := http.NewRequestWithContext(r.Context(), http.MethodHead, testURL, nil)
|
||||
if err != nil {
|
||||
writeJSON(w, cloudConnectionTestResponse{OK: false, Reachable: false, URL: testURL, Error: err.Error()})
|
||||
return
|
||||
}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
writeJSON(w, cloudConnectionTestResponse{OK: true, Reachable: false, URL: testURL, Error: err.Error()})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
reachable := resp.StatusCode > 0 && resp.StatusCode < 500
|
||||
writeJSON(w, cloudConnectionTestResponse{OK: true, Reachable: reachable, URL: testURL, Status: resp.StatusCode})
|
||||
}
|
||||
|
||||
func cloudTemplatePaths(template string) (subdir, zipName string, err error) {
|
||||
switch template {
|
||||
case "s3-cloudfront", "s3_cloudfront":
|
||||
return "s3-cloudfront", "aetherforge-s3-cloudfront.zip", nil
|
||||
case "ssm-document", "ssm_document":
|
||||
return "ssm-document", "aetherforge-ssm-document.zip", nil
|
||||
case "launch-template", "launch_template":
|
||||
return "launch-template", "aetherforge-launch-template.zip", nil
|
||||
case "fargate", "fargate-burst", "fargate_burst":
|
||||
return "fargate", "aetherforge-fargate.zip", nil
|
||||
case "eventbridge", "event-bridge":
|
||||
return "eventbridge", "aetherforge-eventbridge.zip", nil
|
||||
case "cloud-map", "cloud_map":
|
||||
return "cloud-map", "aetherforge-cloud-map.zip", nil
|
||||
case "minio":
|
||||
return "minio", "aetherforge-minio.zip", nil
|
||||
case "curl-manifest", "curl_manifest":
|
||||
return "curl-manifest", "aetherforge-curl-manifest.zip", nil
|
||||
default:
|
||||
return "", "", fmt.Errorf("unknown cloud template %q", template)
|
||||
}
|
||||
}
|
||||
|
||||
func cloudTestURL(kind, endpoint, bucket string) (string, error) {
|
||||
switch kind {
|
||||
case "http", "https", "curl", "cloudfront":
|
||||
if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
return endpoint, nil
|
||||
case "s3", "aws-s3":
|
||||
if bucket == "" {
|
||||
return "", fmt.Errorf("bucket required for s3 test")
|
||||
}
|
||||
base := endpoint
|
||||
if base == "" {
|
||||
base = "https://s3.amazonaws.com"
|
||||
}
|
||||
if !strings.HasPrefix(base, "http://") && !strings.HasPrefix(base, "https://") {
|
||||
base = "https://" + base
|
||||
}
|
||||
return strings.TrimRight(base, "/") + "/" + bucket, nil
|
||||
case "minio", "s3-compatible":
|
||||
if bucket == "" {
|
||||
return "", fmt.Errorf("bucket required for minio test")
|
||||
}
|
||||
if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
|
||||
endpoint = "https://" + endpoint
|
||||
}
|
||||
return strings.TrimRight(endpoint, "/") + "/" + bucket, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown connection kind %q", kind)
|
||||
}
|
||||
}
|
||||
72
server/internal/api/cloud_spread_test.go
Normal file
72
server/internal/api/cloud_spread_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExportCloudTemplateZIP(t *testing.T) {
|
||||
root := integrationWorkspaceRoot(t)
|
||||
h := NewSpreadHandler(nil, t.TempDir(), root, nil)
|
||||
body, _ := json.Marshal(map[string]interface{}{
|
||||
"template": "curl-manifest",
|
||||
"server_url": "https://deck.example",
|
||||
"build_id": "pin-cloud",
|
||||
"campaign": "aws-wave",
|
||||
"bucket": "lab-shards",
|
||||
"cloudfront_domain": "d111.cloudfront.net",
|
||||
"minio_endpoint": "https://minio.lab:9000",
|
||||
"region": "us-west-2",
|
||||
})
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/builder/cloud-template-export", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
h.ExportCloudTemplate(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
entries := readZipEntries(t, rec.Body.Bytes())
|
||||
if !strings.Contains(entries["manifest.json"], "https://deck.example") {
|
||||
t.Fatalf("manifest.json: %s", entries["manifest.json"])
|
||||
}
|
||||
if !strings.Contains(entries["manifest.json"], "pin-cloud") {
|
||||
t.Fatalf("expected build pin in manifest: %s", entries["manifest.json"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportCloudTemplateRequiresServerURL(t *testing.T) {
|
||||
root := integrationWorkspaceRoot(t)
|
||||
h := NewSpreadHandler(nil, t.TempDir(), root, nil)
|
||||
body, _ := json.Marshal(map[string]string{"template": "minio"})
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/builder/cloud-template-export", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
h.ExportCloudTemplate(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloudConnectionTestHTTP(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
h := NewSpreadHandler(nil, t.TempDir(), ".", nil)
|
||||
body, _ := json.Marshal(map[string]string{"kind": "http", "endpoint": srv.URL})
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/builder/cloud-connection-test", bytes.NewReader(body))
|
||||
rec := httptest.NewRecorder()
|
||||
h.TestCloudConnection(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var resp cloudConnectionTestResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !resp.OK || !resp.Reachable {
|
||||
t.Fatalf("resp=%+v", resp)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,10 @@ type ServerPolicy struct {
|
||||
ErasureLanesEnabled bool
|
||||
// FleetTorrentEnabled enables fleet-wide shard DHT gossip and torrent manifests.
|
||||
FleetTorrentEnabled bool
|
||||
AwsS3ShardRegion string
|
||||
AwsCloudFrontDomain string
|
||||
FargateBurstCampaign bool
|
||||
FargateBurstTTLHours int
|
||||
// StrainHospiceWinRateThreshold auto-retires strains below this win rate when AI control is on.
|
||||
StrainHospiceWinRateThreshold float64
|
||||
// StrainHospiceMinAttempts is minimum spread outcomes before AI auto-hospice applies.
|
||||
|
||||
@@ -23,7 +23,10 @@ type SpreadHandler struct {
|
||||
dataDir string
|
||||
projectRoot string
|
||||
wsHub *WSHub
|
||||
publicURL func() string
|
||||
erasureShards *erasure.ShardStore
|
||||
deployPlan *DeployPlanHandler
|
||||
s3CRRConfigFn func() erasure.S3ShardConfig
|
||||
notesMu sync.RWMutex
|
||||
}
|
||||
|
||||
@@ -33,6 +36,36 @@ func (h *SpreadHandler) BindErasureShards(store *erasure.ShardStore) {
|
||||
}
|
||||
}
|
||||
|
||||
func (h *SpreadHandler) BindDeployPlan(handler *DeployPlanHandler) {
|
||||
if h != nil {
|
||||
h.deployPlan = handler
|
||||
}
|
||||
}
|
||||
|
||||
func (h *SpreadHandler) BindS3CRRConfig(fn func() erasure.S3ShardConfig) {
|
||||
if h != nil {
|
||||
h.s3CRRConfigFn = fn
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/v1/spread/aws-s3-crr-template — operator-applied CRR JSON (no AWS API calls).
|
||||
func (h *SpreadHandler) GetS3CRRTemplate(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.s3CRRConfigFn == nil {
|
||||
http.Error(w, "s3 crr not configured", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
doc, err := erasure.BuildS3CRRRule(h.s3CRRConfigFn())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]interface{}{
|
||||
"template": "templates/spread/aws/s3-crr-rule.json",
|
||||
"rule": doc,
|
||||
"notes": "Apply via S3 console or CLI; enables cross-region shard epidemic replication under shards/",
|
||||
})
|
||||
}
|
||||
|
||||
func NewSpreadHandler(database *dbpkg.Database, dataDir, projectRoot string, wsHub *WSHub) *SpreadHandler {
|
||||
return &SpreadHandler{db: database, dataDir: dataDir, projectRoot: projectRoot, wsHub: wsHub}
|
||||
}
|
||||
|
||||
33
server/internal/api/spread_s3_crr_test.go
Normal file
33
server/internal/api/spread_s3_crr_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"crypto-miner-server/internal/erasure"
|
||||
)
|
||||
|
||||
func TestGetS3CRRTemplate(t *testing.T) {
|
||||
h := NewSpreadHandler(nil, t.TempDir(), t.TempDir(), nil)
|
||||
h.BindS3CRRConfig(func() erasure.S3ShardConfig {
|
||||
return erasure.S3ShardConfig{
|
||||
ShardBucket: "primary", ShardRegion: "us-east-1",
|
||||
CRRDestBucket: "replica", CRRDestRegion: "eu-west-1",
|
||||
}
|
||||
})
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/spread/aws-s3-crr-template", nil)
|
||||
w := httptest.NewRecorder()
|
||||
h.GetS3CRRTemplate(w, req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body["rule"] == nil {
|
||||
t.Fatalf("body=%v", body)
|
||||
}
|
||||
}
|
||||
@@ -179,6 +179,9 @@ type WSHub struct {
|
||||
epidemiology *epidemiology.Tracker
|
||||
miningSurgery *miningsurgery.Tracker
|
||||
contingencyOrch *mining.ContingencyOrchestrator
|
||||
fargateBurstCampaign bool
|
||||
fargateBurstExpiresAt time.Time
|
||||
fargateBurstTTLHours int
|
||||
pingIntervalSec int
|
||||
fleetSecret string // baked into forged agents; verified on WS connect
|
||||
eventNotifier *alerts.Notifier
|
||||
|
||||
41
server/internal/cloudmap/registry.go
Normal file
41
server/internal/cloudmap/registry.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package cloudmap
|
||||
|
||||
import "strings"
|
||||
|
||||
type RegistryDocument struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Service string `json:"service"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
Instances []RegistryInstance `json:"instances"`
|
||||
}
|
||||
|
||||
type RegistryInstance struct {
|
||||
AgentID string `json:"agent_id"`
|
||||
DNSName string `json:"dns_name"`
|
||||
IP string `json:"ip"`
|
||||
FetchURL string `json:"fetch_url"`
|
||||
Port int `json:"port"`
|
||||
Healthy bool `json:"healthy"`
|
||||
}
|
||||
|
||||
func SeederDNSName(service, namespace string) string {
|
||||
if strings.TrimSpace(service) == "" {
|
||||
service = "seeder"
|
||||
}
|
||||
if strings.TrimSpace(namespace) == "" {
|
||||
namespace = "prod.local"
|
||||
}
|
||||
return strings.TrimSpace(service) + ".svc." + strings.TrimSpace(namespace)
|
||||
}
|
||||
|
||||
func NormalizeRegistryDocument(doc RegistryDocument) (RegistryDocument, bool) {
|
||||
doc.Namespace = strings.TrimSpace(doc.Namespace)
|
||||
doc.Service = strings.TrimSpace(doc.Service)
|
||||
if doc.Namespace == "" {
|
||||
doc.Namespace = "prod.local"
|
||||
}
|
||||
if doc.Service == "" {
|
||||
doc.Service = "seeder"
|
||||
}
|
||||
return doc, doc.Namespace != "" && doc.Service != ""
|
||||
}
|
||||
9
server/internal/cloudmap/registry_test.go
Normal file
9
server/internal/cloudmap/registry_test.go
Normal file
@@ -0,0 +1,9 @@
|
||||
package cloudmap
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSeederDNSName(t *testing.T) {
|
||||
if got := SeederDNSName("seeder", "prod.local"); got != "seeder.svc.prod.local" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
79
server/internal/erasure/s3_crr.go
Normal file
79
server/internal/erasure/s3_crr.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package erasure
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type S3ShardConfig struct {
|
||||
ShardBucket string `json:"aws_s3_shard_bucket"`
|
||||
ShardRegion string `json:"aws_s3_shard_region"`
|
||||
CRRDestBucket string `json:"aws_s3_crr_dest_bucket"`
|
||||
CRRDestRegion string `json:"aws_s3_crr_dest_region"`
|
||||
CloudFrontDomain string `json:"aws_cloudfront_domain,omitempty"`
|
||||
ReplicationRoleARN string `json:"aws_s3_replication_role_arn,omitempty"`
|
||||
ReplicationAccountID string `json:"aws_account_id,omitempty"`
|
||||
}
|
||||
|
||||
func S3ShardKey(token, region string, shardIndex int, shardHash string) string {
|
||||
token = strings.TrimSpace(token)
|
||||
region = strings.TrimSpace(region)
|
||||
shardHash = strings.TrimSpace(strings.ToLower(shardHash))
|
||||
if token == "" || region == "" || shardHash == "" {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("shards/%s/%s/%d-%s.bin", token, region, shardIndex, shardHash)
|
||||
}
|
||||
|
||||
func S3ShardMetadata(region string, shardIndex int, token, shardHash string) map[string]string {
|
||||
return map[string]string{
|
||||
"region": strings.TrimSpace(region),
|
||||
"shard-index": fmt.Sprintf("%d", shardIndex),
|
||||
"token": strings.TrimSpace(token),
|
||||
"shard-hash": strings.TrimSpace(strings.ToLower(shardHash)),
|
||||
"shard-advert": FormatShardAdvert(region, shardIndex),
|
||||
}
|
||||
}
|
||||
|
||||
func FormatShardAdvert(region string, shardIndex int) string {
|
||||
return fmt.Sprintf("%s:%d", strings.TrimSpace(region), shardIndex)
|
||||
}
|
||||
|
||||
func ParseShardAdvert(advert string) (region string, index int, ok bool) {
|
||||
advert = strings.TrimSpace(advert)
|
||||
colon := strings.LastIndex(advert, ":")
|
||||
if colon <= 0 {
|
||||
return "", 0, false
|
||||
}
|
||||
region = strings.TrimSpace(advert[:colon])
|
||||
if _, err := fmt.Sscanf(strings.TrimSpace(advert[colon+1:]), "%d", &index); err != nil {
|
||||
return "", 0, false
|
||||
}
|
||||
return region, index, region != ""
|
||||
}
|
||||
|
||||
func BuildS3CRRRule(cfg S3ShardConfig) (map[string]interface{}, error) {
|
||||
if cfg.ShardBucket == "" || cfg.ShardRegion == "" || cfg.CRRDestBucket == "" || cfg.CRRDestRegion == "" {
|
||||
return nil, fmt.Errorf("aws s3 shard source/dest bucket and regions required")
|
||||
}
|
||||
role := strings.TrimSpace(cfg.ReplicationRoleARN)
|
||||
if role == "" {
|
||||
acct := strings.TrimSpace(cfg.ReplicationAccountID)
|
||||
if acct == "" {
|
||||
acct = "ACCOUNT_ID"
|
||||
}
|
||||
role = fmt.Sprintf("arn:aws:iam::%s:role/aetherforge-s3-shard-replication", acct)
|
||||
}
|
||||
dest := strings.TrimSpace(cfg.CRRDestBucket)
|
||||
if !strings.HasPrefix(dest, "arn:") {
|
||||
dest = "arn:aws:s3:::" + dest
|
||||
}
|
||||
return map[string]interface{}{
|
||||
"Role": role,
|
||||
"Rules": []map[string]interface{}{{
|
||||
"ID": "aetherforge-erasure-shard-crr", "Status": "Enabled", "Priority": 1,
|
||||
"Filter": map[string]interface{}{"Prefix": "shards/"},
|
||||
"Destination": map[string]interface{}{"Bucket": dest},
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
33
server/internal/erasure/s3_crr_test.go
Normal file
33
server/internal/erasure/s3_crr_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package erasure
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestS3ShardKeyRegional(t *testing.T) {
|
||||
if got := S3ShardKey("tok", "eu-west-1", 2, "abc"); got != "shards/tok/eu-west-1/2-abc.bin" {
|
||||
t.Fatalf("key=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatParseShardAdvert(t *testing.T) {
|
||||
a := FormatShardAdvert("us-east-1", 3)
|
||||
r, i, ok := ParseShardAdvert(a)
|
||||
if !ok || r != "us-east-1" || i != 3 {
|
||||
t.Fatalf("r=%q i=%d", r, i)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildS3CRRRule(t *testing.T) {
|
||||
doc, err := BuildS3CRRRule(S3ShardConfig{
|
||||
ShardBucket: "p", ShardRegion: "us-east-1", CRRDestBucket: "r", CRRDestRegion: "eu-west-1",
|
||||
ReplicationAccountID: "111",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if role, _ := doc["Role"].(string); !strings.Contains(role, "111") {
|
||||
t.Fatalf("role=%q", role)
|
||||
}
|
||||
}
|
||||
108
server/internal/fargate/burst.go
Normal file
108
server/internal/fargate/burst.go
Normal file
@@ -0,0 +1,108 @@
|
||||
// Package fargate generates standalone ECS Fargate burst-seeder task bundles.
|
||||
package fargate
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultTaskFamily = "aetherforge-burst-seeder"
|
||||
DefaultImage = "public.ecr.aws/docker/library/nginx:alpine"
|
||||
DefaultRegion = "us-east-1"
|
||||
DefaultCluster = "aetherforge-burst"
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
BuildID, Campaign, ServerURL, ShardToken, PayloadSHA256 string
|
||||
PayloadSize, DataShards, ParityShards, TTLHours, TaskCount int
|
||||
ShardURLs []string
|
||||
SwarmMagnet string
|
||||
Shards [][]byte
|
||||
Region, Cluster, TaskFamily, Image string
|
||||
}
|
||||
|
||||
type Bundle struct {
|
||||
TaskDefinitionJSON, RunTaskScript, ShardManifestJSON []byte
|
||||
ShardToken string
|
||||
}
|
||||
|
||||
func GenerateBundle(opts Options) (*Bundle, error) {
|
||||
opts = opts.withDefaults()
|
||||
if len(opts.Shards) == 0 {
|
||||
return nil, fmt.Errorf("fargate: shards required")
|
||||
}
|
||||
manifest, err := json.MarshalIndent(map[string]interface{}{
|
||||
"token": opts.ShardToken, "shards_b64": encodeShards(opts.Shards), "ttl_hours": opts.TTLHours,
|
||||
}, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskDef, err := json.MarshalIndent(map[string]interface{}{
|
||||
"family": opts.TaskFamily, "networkMode": "awsvpc", "requiresCompatibilities": []string{"FARGATE"},
|
||||
"cpu": "256", "memory": "512",
|
||||
"containerDefinitions": []map[string]interface{}{{
|
||||
"name": "burst-seeder", "image": opts.Image, "essential": true,
|
||||
"environment": []map[string]string{
|
||||
{"name": "AF_SHARD_TOKEN", "value": opts.ShardToken},
|
||||
{"name": "AF_SHARD_MANIFEST_B64", "value": base64.StdEncoding.EncodeToString(manifest)},
|
||||
},
|
||||
}},
|
||||
}, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
script := fmt.Sprintf(`#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
aws ecs register-task-definition --region "${AWS_REGION:-%s}" --cli-input-json file://task-definition.json
|
||||
aws ecs run-task --region "${AWS_REGION:-%s}" --cluster "${ECS_CLUSTER:-%s}" --launch-type FARGATE --task-definition %s --count ${ECS_TASK_COUNT:-%d}
|
||||
`, opts.Region, opts.Region, opts.Cluster, opts.TaskFamily, opts.TaskCount)
|
||||
return &Bundle{TaskDefinitionJSON: taskDef, RunTaskScript: []byte(script), ShardManifestJSON: manifest, ShardToken: opts.ShardToken}, nil
|
||||
}
|
||||
|
||||
func (o Options) withDefaults() Options {
|
||||
if o.TaskFamily == "" {
|
||||
o.TaskFamily = DefaultTaskFamily
|
||||
}
|
||||
if o.Image == "" {
|
||||
o.Image = DefaultImage
|
||||
}
|
||||
if o.Region == "" {
|
||||
o.Region = DefaultRegion
|
||||
}
|
||||
if o.Cluster == "" {
|
||||
o.Cluster = DefaultCluster
|
||||
}
|
||||
o.TTLHours = ClampTTLHours(o.TTLHours)
|
||||
if o.TaskCount <= 0 {
|
||||
o.TaskCount = 3
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func encodeShards(shards [][]byte) []string {
|
||||
out := make([]string, len(shards))
|
||||
for i, sh := range shards {
|
||||
out[i] = base64.StdEncoding.EncodeToString(sh)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func ClampTTLHours(h int) int {
|
||||
if h < 2 {
|
||||
return 2
|
||||
}
|
||||
if h > 4 {
|
||||
return 4
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func BurstTaskDefinition(opts Options) ([]byte, error) {
|
||||
b, err := GenerateBundle(opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.TaskDefinitionJSON, nil
|
||||
}
|
||||
16
server/internal/fargate/burst_test.go
Normal file
16
server/internal/fargate/burst_test.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package fargate
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerateBundle(t *testing.T) {
|
||||
b, err := GenerateBundle(Options{ShardToken: "t", Shards: [][]byte{[]byte("x")}, TTLHours: 3})
|
||||
if err != nil || len(b.TaskDefinitionJSON) == 0 {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if !strings.Contains(string(b.TaskDefinitionJSON), "AF_SHARD_MANIFEST_B64") {
|
||||
t.Fatal("missing manifest env")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user