AetherForge/cloudflare/tunnel-watch.ps1

param(
    [Parameter(Mandatory = $true)][string]$Root
)

$ErrorActionPreference = 'Continue'
$Root = $Root.TrimEnd('\')

$configPath = Join-Path $Root 'cloudflare\tunnel-config.json'
$repairScript = Join-Path $Root 'cloudflare\repair-tunnel.ps1'
$logDir = Join-Path $Root 'data\logs'
$logFile = Join-Path $logDir 'tunnel-watch.log'
$stateFile = Join-Path $env:ProgramData 'AetherForge\tunnel-watch-state.json'
$lockFile = Join-Path $env:ProgramData 'AetherForge\tunnel-watch.pid'

if (-not (Test-Path $configPath)) {
    Write-Host "[CF-WATCH] ERROR: missing $configPath"
    exit 1
}

$cfg = Get-Content -LiteralPath $configPath -Raw | ConvertFrom-Json
$hostname = [string]$cfg.hostname
$origin = [string]$cfg.origin
$interval = if ($cfg.watch_interval_seconds) { [int]$cfg.watch_interval_seconds } else { 60 }
$publicTimeout = if ($cfg.public_check_timeout_seconds) { [int]$cfg.public_check_timeout_seconds } else { 20 }
$originTimeout = if ($cfg.origin_check_timeout_seconds) { [int]$cfg.origin_check_timeout_seconds } else { 8 }
$healCfg = $cfg.heal_after_failures
$lvlRestart = if ($healCfg.restart_service) { [int]$healCfg.restart_service } else { 1 }
$lvlRewrite = if ($healCfg.rewrite_config_and_restart) { [int]$healCfg.rewrite_config_and_restart } else { 2 }
$lvlReinstall = if ($healCfg.full_reinstall) { [int]$healCfg.full_reinstall } else { 4 }

New-Item -ItemType Directory -Force -Path $logDir | Out-Null
New-Item -ItemType Directory -Force -Path (Split-Path $stateFile) | Out-Null
Set-Content -LiteralPath $lockFile -Value $PID -Encoding ASCII

function Write-WatchLog([string]$Message, [string]$Level = 'INFO') {
    $line = '{0} [CF-WATCH] [{1}] {2}' -f (Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $Level, $Message
    Write-Host $line
    Add-Content -LiteralPath $logFile -Value $line -Encoding UTF8
}

function Get-State {
    if (Test-Path $stateFile) {
        try { return Get-Content $stateFile -Raw | ConvertFrom-Json } catch { }
    }
    return [pscustomobject]@{
        consecutive_failures = 0
        last_heal_level      = 0
        last_heal_at         = $null
        last_ok              = $null
    }
}

function Set-State($state) {
    $state | ConvertTo-Json | Set-Content -LiteralPath $stateFile -Encoding UTF8
}

function Test-ServiceRunning {
    $q = sc.exe query cloudflared 2>&1 | Out-String
    if ($q -match '1060|does not exist') { return 'MISSING' }
    if ($q -match 'STATE\s+:\s+\d+\s+RUNNING') { return 'RUNNING' }
    if ($q -match 'STATE\s+:\s+\d+\s+STOPPED') { return 'STOPPED' }
    return 'UNKNOWN'
}

function Test-Http([string]$Url, [int]$TimeoutSec) {
    try {
        $r = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec $TimeoutSec
        return [pscustomobject]@{ Ok = $true; Code = [int]$r.StatusCode; Error = $null }
    } catch {
        $code = $null
        if ($_.Exception.Response) { $code = [int]$_.Exception.Response.StatusCode }
        return [pscustomobject]@{ Ok = $false; Code = $code; Error = $_.Exception.Message }
    }
}

function Invoke-Heal([int]$Level) {
    Write-WatchLog "Self-heal level $Level triggered" 'HEAL'
    if (-not (Test-Path $repairScript)) {
        Write-WatchLog "repair-tunnel.ps1 missing — cannot heal" 'ERROR'
        return $false
    }
    $out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $repairScript -Root $Root -Level $Level 2>&1
    foreach ($line in $out) { Write-WatchLog $line 'HEAL' }
    return ($LASTEXITCODE -eq 0)
}

Write-WatchLog "Watchdog started — interval ${interval}s | public=https://$hostname/ | origin=$origin"
Write-WatchLog "Log file: $logFile"

while ($true) {
    $state = Get-State
    $svc = Test-ServiceRunning
    $originCheck = Test-Http ($origin.TrimEnd('/') + '/') $originTimeout
    $publicCheck = Test-Http ("https://$hostname/") $publicTimeout

    $originLabel = if ($originCheck.Ok) { "OK $($originCheck.Code)" } else { "FAIL $($originCheck.Error)" }
    $publicLabel = if ($publicCheck.Ok) { "OK $($publicCheck.Code)" } else { "FAIL $($publicCheck.Error)" }

    $allOk = ($svc -eq 'RUNNING') -and $originCheck.Ok -and $publicCheck.Ok

    if ($allOk) {
        if ($state.consecutive_failures -gt 0) {
            Write-WatchLog "Recovered — service=$svc origin=$originLabel public=$publicLabel" 'OK'
        } else {
            Write-WatchLog "OK service=$svc origin=$originLabel public=$publicLabel" 'OK'
        }
        Set-State ([pscustomobject]@{
            consecutive_failures = 0
            last_heal_level      = 0
            last_heal_at         = $null
            last_ok              = (Get-Date).ToString('o')
        })
    } else {
        $state.consecutive_failures = [int]$state.consecutive_failures + 1
        $fail = $state.consecutive_failures
        Write-WatchLog "DEGRADED (#$fail) service=$svc origin=$originLabel public=$publicLabel" 'WARN'

        $healLevel = 0
        if ($svc -eq 'MISSING' -or $fail -ge $lvlReinstall) {
            $healLevel = 4
        } elseif ($svc -ne 'RUNNING' -and $fail -ge $lvlRestart) {
            $healLevel = if ($fail -ge $lvlRewrite) { 3 } else { 1 }
        } elseif ($svc -eq 'RUNNING' -and $originCheck.Ok -and -not $publicCheck.Ok) {
            if ($fail -ge $lvlReinstall) { $healLevel = 4 }
            elseif ($fail -ge $lvlRewrite) { $healLevel = 3 }
            elseif ($fail -ge $lvlRestart) { $healLevel = 2 }
        } elseif ($svc -eq 'RUNNING' -and -not $originCheck.Ok) {
            Write-WatchLog "Origin down — start/repair AetherForge on $origin (tunnel cannot heal dead origin)" 'WARN'
        }

        if ($healLevel -gt 0) {
            $skipHeal = $false
            if ($healLevel -eq 4 -and [int]$state.last_heal_level -eq 4 -and $state.last_heal_at) {
                try {
                    $elapsed = (Get-Date) - [datetime]$state.last_heal_at
                    if ($elapsed.TotalMinutes -lt 10) {
                        Write-WatchLog "Reinstall cooldown (10 min) — skipping level 4 retry" 'WARN'
                        $skipHeal = $true
                    }
                } catch { }
            }
            if (-not $skipHeal) {
                $healed = Invoke-Heal $healLevel
                $state.last_heal_level = $healLevel
                $state.last_heal_at = (Get-Date).ToString('o')
                if ($healed) {
                    Write-WatchLog "Heal level $healLevel completed — recheck next interval" 'HEAL'
                } else {
                    Write-WatchLog "Heal level $healLevel failed — will retry on next check" 'ERROR'
                }
            }
        }

        Set-State $state
    }

    Start-Sleep -Seconds $interval
}