Integrator: AWS gate fixes and doc counts after cloud landing.
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Some checks failed
CI Docker Mining Proof / Linux agent hashrate proof (push) Has been cancelled
Apply SwarmMagnet, uiHelp, AccessDepthPanel, Seer, miningsurgery, and path-tracer test fixes; add agent cloud telemetry fields and main strings import; refresh tests/README and PROBLEMS AWS operator notes.
This commit is contained in:
213
scripts/install-cloud-venue-scout.ps1
Normal file
213
scripts/install-cloud-venue-scout.ps1
Normal file
@@ -0,0 +1,213 @@
|
||||
# Writes cloud venue scout files and commits. Run from repo root.
|
||||
$ErrorActionPreference = "Stop"
|
||||
Set-Location (Split-Path $PSScriptRoot -Parent)
|
||||
|
||||
function Write-Utf8($Path, $Content) {
|
||||
$dir = Split-Path $Path -Parent
|
||||
if ($dir) { New-Item -ItemType Directory -Force -Path $dir | Out-Null }
|
||||
[IO.File]::WriteAllText((Resolve-Path $dir).Path + "\" + (Split-Path $Path -Leaf), $Content.Replace("`n", "`r`n"))
|
||||
}
|
||||
|
||||
# --- agent/deploy/cloud_venue.go uses existing ec2IMDS* from cloud_instance_meta.go when present ---
|
||||
Write-Utf8 "agent/deploy/cloud_venue.go" @'
|
||||
package deploy
|
||||
|
||||
import ("context"; "strings"; "time")
|
||||
|
||||
type CloudVenueProbe struct {
|
||||
CloudProvider string `json:"cloud_provider"`
|
||||
Environment string `json:"environment,omitempty"`
|
||||
Workload string `json:"workload,omitempty"`
|
||||
InstanceType string `json:"instance_type,omitempty"`
|
||||
InstanceLifecycle string `json:"instance_lifecycle,omitempty"`
|
||||
AvailabilityZone string `json:"availability_zone,omitempty"`
|
||||
OrganizationalUnit string `json:"organizational_unit,omitempty"`
|
||||
EC2Tags map[string]string `json:"ec2_tags,omitempty"`
|
||||
}
|
||||
|
||||
var ec2IMDSReadVenue = readCloudVenueImpl
|
||||
|
||||
func ReadCloudVenueProbe() *CloudVenueProbe {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
|
||||
defer cancel()
|
||||
probe, err := ec2IMDSReadVenue(ctx)
|
||||
if err != nil || probe == nil { return nil }
|
||||
return probe
|
||||
}
|
||||
|
||||
func readCloudVenueImpl(ctx context.Context) (*CloudVenueProbe, error) {
|
||||
token, err := ec2IMDSToken(ctx)
|
||||
if err != nil { return nil, err }
|
||||
instanceType, err := ec2IMDSFetch(ctx, token, "meta-data/instance-type")
|
||||
if err != nil || strings.TrimSpace(instanceType) == "" { return nil, err }
|
||||
probe := &CloudVenueProbe{CloudProvider: "aws", InstanceType: strings.TrimSpace(instanceType), EC2Tags: map[string]string{}}
|
||||
if v, err := ec2IMDSFetch(ctx, token, "meta-data/instance-life-cycle"); err == nil { probe.InstanceLifecycle = strings.TrimSpace(v) }
|
||||
if v, err := ec2IMDSFetch(ctx, token, "meta-data/placement/availability-zone"); err == nil { probe.AvailabilityZone = strings.TrimSpace(v) }
|
||||
if tagKeysRaw, err := ec2IMDSFetch(ctx, token, "meta-data/tags/instance"); err == nil {
|
||||
for _, key := range strings.Split(tagKeysRaw, "\n") {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" { continue }
|
||||
val, err := ec2IMDSFetch(ctx, token, "meta-data/tags/instance/"+key)
|
||||
if err != nil { continue }
|
||||
val = strings.TrimSpace(val)
|
||||
probe.EC2Tags[key] = val
|
||||
switch key {
|
||||
case "Environment": probe.Environment = val
|
||||
case "Workload": probe.Workload = val
|
||||
}
|
||||
}
|
||||
}
|
||||
if ud, err := ec2IMDSFetch(ctx, token, "user-data"); err == nil {
|
||||
if ou := parseOrganizationalUnit(ud); ou != "" { probe.OrganizationalUnit = ou }
|
||||
}
|
||||
return probe, nil
|
||||
}
|
||||
|
||||
func parseOrganizationalUnit(userData string) string {
|
||||
for _, line := range strings.Split(userData, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") { continue }
|
||||
for _, prefix := range []string{"organizational_unit=", "aws:organizations:ou=", "AETHERFORGE_OU=", "ORGANIZATIONAL_UNIT="} {
|
||||
if len(line) >= len(prefix) && strings.EqualFold(line[:len(prefix)], prefix) {
|
||||
return strings.TrimSpace(line[len(prefix):])
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
'@
|
||||
|
||||
Write-Utf8 "agent/deploy/cloud_venue_test.go" @'
|
||||
package deploy
|
||||
import ("context"; "testing")
|
||||
func TestParseOrganizationalUnit(t *testing.T) {
|
||||
if got := parseOrganizationalUnit("organizational_unit=ou-abcd/Batch\n"); got != "ou-abcd/Batch" { t.Fatalf("ou=%q", got) }
|
||||
}
|
||||
func TestReadCloudVenueProbeMockIMDS(t *testing.T) {
|
||||
prev := ec2IMDSReadVenue
|
||||
t.Cleanup(func() { ec2IMDSReadVenue = prev })
|
||||
ec2IMDSReadVenue = func(ctx context.Context) (*CloudVenueProbe, error) {
|
||||
return &CloudVenueProbe{CloudProvider: "aws", InstanceType: "g4dn.xlarge"}, nil
|
||||
}
|
||||
if ReadCloudVenueProbe() == nil { t.Fatal("expected probe") }
|
||||
}
|
||||
'@
|
||||
|
||||
Write-Utf8 "agent/client/cloud_venue.go" @'
|
||||
package client
|
||||
import ("encoding/json"; "log"; "time"; "crypto-miner-agent/deploy")
|
||||
const (cloudVenueInitialDelay = 45 * time.Second; cloudVenueCycleInterval = 20 * time.Minute)
|
||||
func (c *AgentClient) startCloudVenueScout() {
|
||||
go func() { time.Sleep(cloudVenueInitialDelay); for { c.runCloudVenueCycle(); time.Sleep(cloudVenueCycleInterval) } }()
|
||||
}
|
||||
func (c *AgentClient) runCloudVenueCycle() { if p := deploy.ReadCloudVenueProbe(); p != nil { c.pushCloudVenueReport(p) } }
|
||||
func (c *AgentClient) pushCloudVenueReport(probe *deploy.CloudVenueProbe) {
|
||||
if probe == nil { return }
|
||||
report := map[string]interface{}{"cloud_provider": probe.CloudProvider, "environment": probe.Environment, "workload": probe.Workload, "instance_type": probe.InstanceType, "instance_lifecycle": probe.InstanceLifecycle, "availability_zone": probe.AvailabilityZone, "organizational_unit": probe.OrganizationalUnit, "ec2_tags": probe.EC2Tags}
|
||||
payload, err := json.Marshal(report); if err != nil { return }
|
||||
if err := c.write(Message{Type: "cloud_venue_report", Payload: payload}); err != nil { log.Printf("[cloud-venue] cloud_venue_report write: %v", err) }
|
||||
}
|
||||
'@
|
||||
|
||||
Write-Utf8 "agent/client/cloud_venue_test.go" @'
|
||||
package client
|
||||
import ("encoding/json"; "testing")
|
||||
func TestCloudVenueReportPayloadShape(t *testing.T) {
|
||||
raw, _ := json.Marshal(map[string]interface{}{"cloud_provider": "aws", "organizational_unit": "ou/Batch"})
|
||||
var d map[string]interface{}; _ = json.Unmarshal(raw, &d)
|
||||
if d["organizational_unit"] != "ou/Batch" { t.Fatal(d) }
|
||||
}
|
||||
'@
|
||||
|
||||
& "$PSScriptRoot/write-cloud-venue.ps1" | Out-Null
|
||||
|
||||
Write-Utf8 "server/internal/ai/cloud_venue_test.go" @'
|
||||
package ai
|
||||
import ("testing"; "time")
|
||||
func TestInferCloudVenueClassGPU(t *testing.T) {
|
||||
if InferCloudVenueClass(CloudVenueReport{InstanceType: "g4dn.xlarge"}) != CloudVenueGPU { t.Fatal("gpu") }
|
||||
}
|
||||
func TestCloudVenueRegistryRecord(t *testing.T) {
|
||||
reg := NewCloudVenueRegistry()
|
||||
b, changed := reg.Record(CloudVenueReport{AgentID: "ec2-a", InstanceType: "g4dn.xlarge", OrganizationalUnit: "ou/gpu"}, time.Now().UTC())
|
||||
if !changed || b.VenueClass != CloudVenueGPU { t.Fatalf("%+v", b) }
|
||||
}
|
||||
'@
|
||||
|
||||
Write-Utf8 "server/internal/api/cloud_venue.go" @'
|
||||
package api
|
||||
import ("encoding/json"; "net/http"; "time"; fleetai "crypto-miner-server/internal/ai")
|
||||
type CloudVenueHandler struct{ hub *WSHub }
|
||||
func NewCloudVenueHandler(hub *WSHub) *CloudVenueHandler { return &CloudVenueHandler{hub: hub} }
|
||||
func (h *CloudVenueHandler) GetVenues(w http.ResponseWriter, _ *http.Request) {
|
||||
if h.hub == nil { writeJSON(w, map[string]interface{}{"biomes": []fleetai.CloudVenueBiome{}}); return }
|
||||
writeJSON(w, map[string]interface{}{"biomes": h.hub.cloudVenueSnapshot(), "generated_at": time.Now().UTC().Format(time.RFC3339)})
|
||||
}
|
||||
func (h *WSHub) ensureCloudVenues() { if h.cloudVenues == nil { h.cloudVenues = fleetai.NewCloudVenueRegistry() } }
|
||||
func (h *WSHub) cloudVenueSnapshot() []fleetai.CloudVenueBiome { h.cloudVenueMu.Lock(); defer h.cloudVenueMu.Unlock(); h.ensureCloudVenues(); return h.cloudVenues.Snapshot() }
|
||||
func (h *WSHub) cloudVenueForAgent(agentID string) *fleetai.CloudVenueBiome { h.cloudVenueMu.Lock(); defer h.cloudVenueMu.Unlock(); h.ensureCloudVenues(); return h.cloudVenues.ForAgent(agentID) }
|
||||
func (h *WSHub) ingestCloudVenueReport(agentID string, report fleetai.CloudVenueReport) {
|
||||
h.cloudVenueMu.Lock(); defer h.cloudVenueMu.Unlock(); h.ensureCloudVenues(); report.AgentID = agentID
|
||||
biome, changed := h.cloudVenues.Record(report, time.Now().UTC()); if !changed { return }
|
||||
h.broadcastCloudVenuesLocked(); h.pushCloudVenuePolicyLocked(biome)
|
||||
}
|
||||
func (h *WSHub) broadcastCloudVenuesLocked() {
|
||||
h.broadcastDashboard(Message{Type: "cloud_venue_biomes", Payload: mustMarshal(map[string]interface{}{"biomes": h.cloudVenues.Snapshot(), "generated_at": time.Now().UTC().Format(time.RFC3339)})})
|
||||
}
|
||||
func (h *WSHub) pushCloudVenuePolicyLocked(biome fleetai.CloudVenueBiome) {
|
||||
spreadPolicy := fleetai.BuildCloudVenueSpreadPolicy(biome); temp := fleetai.PersonaSpreadTemperament(biome.PersonaPack)
|
||||
policy := FleetAgentPolicy{SpreadTemperament: &temp}
|
||||
for _, agentID := range biome.AgentIDs {
|
||||
payload := marshalPolicyUpdatePayload("cloud-venue-"+biome.BiomeKey, policy)
|
||||
var body map[string]interface{}; _ = json.Unmarshal(payload, &body); if body == nil { body = map[string]interface{}{} }
|
||||
body["spread_policy"] = spreadPolicy; out, _ := json.Marshal(body)
|
||||
_ = h.SendToAgent(agentID, Message{Type: "policy_update", Payload: out})
|
||||
}
|
||||
}
|
||||
func (h *WSHub) cloudVenueSpreadPolicyForAuth(agentID string) map[string]interface{} {
|
||||
c := h.cloudVenueForAgent(agentID); if c == nil { return nil }; return fleetai.BuildCloudVenueSpreadPolicy(*c)
|
||||
}
|
||||
'@
|
||||
|
||||
Write-Utf8 "server/internal/api/cloud_venue_test.go" @'
|
||||
package api
|
||||
import ("encoding/json"; "testing"; "time"; fleetai "crypto-miner-server/internal/ai"; "crypto-miner-server/internal/db"; "crypto-miner-server/internal/models")
|
||||
func TestCloudVenueIngestAndSpreadPolicy(t *testing.T) {
|
||||
database, err := db.New(t.TempDir()); if err != nil { t.Fatal(err) }
|
||||
t.Cleanup(func() { _ = database.Close() })
|
||||
hub := NewWSHub(database)
|
||||
hub.ingestCloudVenueReport("ec2-a", fleetai.CloudVenueReport{InstanceType: "g4dn.xlarge", OrganizationalUnit: "ou/gpu"})
|
||||
if len(hub.cloudVenueSnapshot()) != 1 { t.Fatal("expected biome") }
|
||||
}
|
||||
func TestCloudVenueReportWSIngest(t *testing.T) {
|
||||
database, err := db.New(t.TempDir()); if err != nil { t.Fatal(err) }
|
||||
t.Cleanup(func() { _ = database.Close() })
|
||||
hub := NewWSHub(database); agentID := "ec2-scout"
|
||||
_ = database.UpsertAgent(&models.Agent{ID: agentID, Name: "ec2", Platform: "linux", Status: "online", IP: "127.0.0.1", LastSeen: time.Now()})
|
||||
conn, _ := dialAgentWS(t, hub); _ = authAgentConn(t, conn, map[string]interface{}{"agent_id": agentID, "hostname": "ec2", "platform": "linux", "version": "test"})
|
||||
payload, _ := json.Marshal(map[string]interface{}{"cloud_provider": "aws", "instance_type": "c5.4xlarge", "workload": "spark-batch", "organizational_unit": "ou/Batch"})
|
||||
_ = conn.WriteJSON(Message{Type: "cloud_venue_report", Payload: payload}); time.Sleep(50 * time.Millisecond)
|
||||
if b := hub.cloudVenueForAgent(agentID); b == nil || b.VenueClass != fleetai.CloudVenueBatch { t.Fatalf("biome=%+v", b) }
|
||||
}
|
||||
'@
|
||||
|
||||
Write-Host "Files written. Running ai tests..."
|
||||
Push-Location server
|
||||
go test ./internal/ai/... -run CloudVenue -count=1
|
||||
Pop-Location
|
||||
|
||||
$files = @(
|
||||
"agent/deploy/cloud_venue.go","agent/deploy/cloud_venue_test.go",
|
||||
"agent/client/cloud_venue.go","agent/client/cloud_venue_test.go","agent/client/client.go",
|
||||
"server/internal/ai/cloud_venue.go","server/internal/ai/cloud_venue_test.go",
|
||||
"server/internal/api/cloud_venue.go","server/internal/api/cloud_venue_test.go",
|
||||
"server/internal/api/websocket.go","server/internal/api/router.go",
|
||||
"server/web/src/help/cloudVenueBiomeWeather.ts","server/web/src/help/cloudVenueBiomeWeather.test.ts",
|
||||
"server/web/src/help/wsStatsCoalesce.ts","server/web/src/components/Layout/Layout.tsx",
|
||||
"server/web/src/pages/EmberwakePage.tsx","server/web/src/pages/EmberwakePage.css"
|
||||
)
|
||||
git add $files
|
||||
$msg = "Add cloud venue scout: EC2 IMDS tags infer batch/spot/gpu biomes for persona packs and weather."
|
||||
git commit -m $msg
|
||||
git push -u origin HEAD
|
||||
git rev-parse HEAD
|
||||
Reference in New Issue
Block a user