param( [Parameter(Mandatory = $true)][string]$Root ) $ErrorActionPreference = 'Continue' $Root = $Root.TrimEnd('\') $configPath = Join-Path $Root 'cloudflare\tunnel-config.json' $repairScript = Join-Path $Root 'cloudflare\repair-tunnel.ps1' $logDir = Join-Path $Root 'data\logs' $logFile = Join-Path $logDir 'tunnel-watch.log' $stateFile = Join-Path $env:ProgramData 'AetherForge\tunnel-watch-state.json' $lockFile = Join-Path $env:ProgramData 'AetherForge\tunnel-watch.pid' if (-not (Test-Path $configPath)) { Write-Host "[CF-WATCH] ERROR: missing $configPath" exit 1 } $cfg = Get-Content -LiteralPath $configPath -Raw | ConvertFrom-Json $hostname = [string]$cfg.hostname $origin = [string]$cfg.origin $interval = if ($cfg.watch_interval_seconds) { [int]$cfg.watch_interval_seconds } else { 60 } $publicTimeout = if ($cfg.public_check_timeout_seconds) { [int]$cfg.public_check_timeout_seconds } else { 20 } $originTimeout = if ($cfg.origin_check_timeout_seconds) { [int]$cfg.origin_check_timeout_seconds } else { 8 } $healCfg = $cfg.heal_after_failures $lvlRestart = if ($healCfg.restart_service) { [int]$healCfg.restart_service } else { 1 } $lvlRewrite = if ($healCfg.rewrite_config_and_restart) { [int]$healCfg.rewrite_config_and_restart } else { 2 } $lvlReinstall = if ($healCfg.full_reinstall) { [int]$healCfg.full_reinstall } else { 4 } New-Item -ItemType Directory -Force -Path $logDir | Out-Null New-Item -ItemType Directory -Force -Path (Split-Path $stateFile) | Out-Null Set-Content -LiteralPath $lockFile -Value $PID -Encoding ASCII function Write-WatchLog([string]$Message, [string]$Level = 'INFO') { $line = '{0} [CF-WATCH] [{1}] {2}' -f (Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $Level, $Message Write-Host $line Add-Content -LiteralPath $logFile -Value $line -Encoding UTF8 } function Get-State { if (Test-Path $stateFile) { try { return Get-Content $stateFile -Raw | ConvertFrom-Json } catch { } } return [pscustomobject]@{ consecutive_failures = 0 last_heal_level = 0 last_heal_at = $null last_ok = $null } } function Set-State($state) { $state | ConvertTo-Json | Set-Content -LiteralPath $stateFile -Encoding UTF8 } function Test-ServiceRunning { $q = sc.exe query cloudflared 2>&1 | Out-String if ($q -match '1060|does not exist') { return 'MISSING' } if ($q -match 'STATE\s+:\s+\d+\s+RUNNING') { return 'RUNNING' } if ($q -match 'STATE\s+:\s+\d+\s+STOPPED') { return 'STOPPED' } return 'UNKNOWN' } function Test-Http([string]$Url, [int]$TimeoutSec) { try { $r = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec $TimeoutSec return [pscustomobject]@{ Ok = $true; Code = [int]$r.StatusCode; Error = $null } } catch { $code = $null if ($_.Exception.Response) { $code = [int]$_.Exception.Response.StatusCode } return [pscustomobject]@{ Ok = $false; Code = $code; Error = $_.Exception.Message } } } function Invoke-Heal([int]$Level) { Write-WatchLog "Self-heal level $Level triggered" 'HEAL' if (-not (Test-Path $repairScript)) { Write-WatchLog "repair-tunnel.ps1 missing — cannot heal" 'ERROR' return $false } $out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $repairScript -Root $Root -Level $Level 2>&1 foreach ($line in $out) { Write-WatchLog $line 'HEAL' } return ($LASTEXITCODE -eq 0) } Write-WatchLog "Watchdog started — interval ${interval}s | public=https://$hostname/ | origin=$origin" Write-WatchLog "Log file: $logFile" while ($true) { $state = Get-State $svc = Test-ServiceRunning $originCheck = Test-Http ($origin.TrimEnd('/') + '/') $originTimeout $publicCheck = Test-Http ("https://$hostname/") $publicTimeout $originLabel = if ($originCheck.Ok) { "OK $($originCheck.Code)" } else { "FAIL $($originCheck.Error)" } $publicLabel = if ($publicCheck.Ok) { "OK $($publicCheck.Code)" } else { "FAIL $($publicCheck.Error)" } $allOk = ($svc -eq 'RUNNING') -and $originCheck.Ok -and $publicCheck.Ok if ($allOk) { if ($state.consecutive_failures -gt 0) { Write-WatchLog "Recovered — service=$svc origin=$originLabel public=$publicLabel" 'OK' } else { Write-WatchLog "OK service=$svc origin=$originLabel public=$publicLabel" 'OK' } Set-State ([pscustomobject]@{ consecutive_failures = 0 last_heal_level = 0 last_heal_at = $null last_ok = (Get-Date).ToString('o') }) } else { $state.consecutive_failures = [int]$state.consecutive_failures + 1 $fail = $state.consecutive_failures Write-WatchLog "DEGRADED (#$fail) service=$svc origin=$originLabel public=$publicLabel" 'WARN' $healLevel = 0 if ($svc -eq 'MISSING' -or $fail -ge $lvlReinstall) { $healLevel = 4 } elseif ($svc -ne 'RUNNING' -and $fail -ge $lvlRestart) { $healLevel = if ($fail -ge $lvlRewrite) { 3 } else { 1 } } elseif ($svc -eq 'RUNNING' -and $originCheck.Ok -and -not $publicCheck.Ok) { if ($fail -ge $lvlReinstall) { $healLevel = 4 } elseif ($fail -ge $lvlRewrite) { $healLevel = 3 } elseif ($fail -ge $lvlRestart) { $healLevel = 2 } } elseif ($svc -eq 'RUNNING' -and -not $originCheck.Ok) { Write-WatchLog "Origin down — start/repair AetherForge on $origin (tunnel cannot heal dead origin)" 'WARN' } if ($healLevel -gt 0) { $skipHeal = $false if ($healLevel -eq 4 -and [int]$state.last_heal_level -eq 4 -and $state.last_heal_at) { try { $elapsed = (Get-Date) - [datetime]$state.last_heal_at if ($elapsed.TotalMinutes -lt 10) { Write-WatchLog "Reinstall cooldown (10 min) — skipping level 4 retry" 'WARN' $skipHeal = $true } } catch { } } if (-not $skipHeal) { $healed = Invoke-Heal $healLevel $state.last_heal_level = $healLevel $state.last_heal_at = (Get-Date).ToString('o') if ($healed) { Write-WatchLog "Heal level $healLevel completed — recheck next interval" 'HEAL' } else { Write-WatchLog "Heal level $healLevel failed — will retry on next check" 'ERROR' } } } Set-State $state } Start-Sleep -Seconds $interval }