From ffbd634706d2a771932b6c933cfb1aae81689809 Mon Sep 17 00:00:00 2001 From: Colin Neilens Date: Mon, 21 Sep 2026 15:08:13 -0700 Subject: [PATCH] Stabilize daemon handoff live test Wait for the required owner/contender supervisor states with liveness checks and timeout diagnostics instead of relying on short fixed polling. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: Colin Neilens --- .../Tests/DaemonHandoff.Live.Tests.ps1 | 113 ++++++++++++++---- Tools/windows/Tests/TrayDaemon.Tests.ps1 | 16 ++- 2 files changed, 106 insertions(+), 23 deletions(-) diff --git a/Tools/windows/Tests/DaemonHandoff.Live.Tests.ps1 b/Tools/windows/Tests/DaemonHandoff.Live.Tests.ps1 index e4b8125f..cba0be69 100644 --- a/Tools/windows/Tests/DaemonHandoff.Live.Tests.ps1 +++ b/Tools/windows/Tests/DaemonHandoff.Live.Tests.ps1 @@ -156,9 +156,77 @@ function Assert-EndpointReachable { $process.Dispose() } +function Get-HandoffElapsed([Diagnostics.Stopwatch] $clock) { + return "{0:n2}s" -f $clock.Elapsed.TotalSeconds +} + +function Get-DaemonTestState([string] $path) { + if (Test-Path -LiteralPath $path) { + return (Get-Content -LiteralPath $path -Raw).Trim() + } + return "" +} + +function Get-ProcessStatus([Diagnostics.Process] $process, [string] $role) { + if (-not $process) { return "$role=" } + $process.Refresh() + return "$role(pid=$($process.Id), exited=$($process.HasExited), exitCode=$( + if ($process.HasExited) { $process.ExitCode } else { '' }))" +} + +function Assert-ShellsAlive( + [Diagnostics.Process] $shellA, + [Diagnostics.Process] $shellB, + [string] $diagnostics +) { + foreach ($entry in @( + @{ Process = $shellA; Role = "shellA" }, + @{ Process = $shellB; Role = "shellB" } + )) { + if (-not $entry.Process) { continue } + $entry.Process.Refresh() + if ($entry.Process.HasExited) { + throw "$($entry.Role) exited before daemon handoff completed: $diagnostics" + } + } +} + +function Get-HandoffDiagnostics( + [Diagnostics.Process] $shellA, + [Diagnostics.Process] $shellB, + [string] $daemonStateA, + [string] $daemonStateB, + [Diagnostics.Stopwatch] $clock, + [Diagnostics.Process] $daemonProcess = $null, + [object] $selectedChild = $null +) { + $ownerState = if ($shellA) { Get-ShellSupervisorState $shellA.Id } else { "" } + $contenderState = if ($shellB) { Get-ShellSupervisorState $shellB.Id } else { "" } + $daemonStatus = if ($daemonProcess) { + $daemonProcess.Refresh() + "daemon(pid=$($daemonProcess.Id), exited=$($daemonProcess.HasExited), exitCode=$( + if ($daemonProcess.HasExited) { $daemonProcess.ExitCode } else { '' }))" + } elseif ($selectedChild) { + "daemon(pid=$($selectedChild.ProcessId), parent=$($selectedChild.ParentProcessId), executable=$($selectedChild.ExecutablePath))" + } else { + "daemon=" + } + return @( + "elapsed=$(Get-HandoffElapsed $clock)", + (Get-ProcessStatus $shellA "shellA"), + (Get-ProcessStatus $shellB "shellB"), + $daemonStatus, + "daemonA=$(Get-DaemonTestState $daemonStateA)", + "daemonB=$(Get-DaemonTestState $daemonStateB)", + "shellStateA=$ownerState", + "shellStateB=$contenderState" + ) -join "; " +} + $shellA = $null $shellB = $null $daemonProcess = $null +$handoffClock = [Diagnostics.Stopwatch]::StartNew() try { New-Item -ItemType Directory -Force -Path $supportDirectory | Out-Null $daemonStateA = Join-Path $supportDirectory "daemon-a.state" @@ -170,7 +238,11 @@ try { $parents = @($shellA.Id, $shellB.Id) $children = @() $selectedChild = $null - for ($i = 0; $i -lt 100; $i++) { + $spawnDeadline = [DateTime]::UtcNow.AddSeconds(20) + while ([DateTime]::UtcNow -lt $spawnDeadline) { + Assert-ShellsAlive $shellA $shellB ( + Get-HandoffDiagnostics $shellA $shellB $daemonStateA $daemonStateB $handoffClock + ) $children = @(Get-DaemonChildren $parents) if ($children.Count -eq 1) { $candidate = Get-Process -Id $children[0].ProcessId -ErrorAction SilentlyContinue @@ -187,27 +259,28 @@ try { Start-Sleep -Milliseconds 100 } if (-not $daemonProcess -or -not $selectedChild) { - $states = @($shellA, $shellB | ForEach-Object { - $_.Refresh() - "pid=$($_.Id), exited=$($_.HasExited), exitCode=$( - if ($_.HasExited) { $_.ExitCode } else { '' })" - }) -join "; " - $daemonStates = @($daemonStateA, $daemonStateB | - ForEach-Object { - "$(Split-Path -Leaf $_)=$(if (Test-Path -LiteralPath $_) { - Get-Content -LiteralPath $_ -Raw - } else { '' })" - }) -join "; " throw ( - "concurrent shells did not spawn exactly one graphcoded child: $states; $daemonStates; " + - "shellStateA=$(Get-ShellSupervisorState $shellA.Id), shellStateB=$(Get-ShellSupervisorState $shellB.Id)" + "concurrent shells did not spawn exactly one graphcoded child before the deadline: " + + (Get-HandoffDiagnostics $shellA $shellB $daemonStateA $daemonStateB $handoffClock) ) } $owner = if ($selectedChild.ParentProcessId -eq $shellA.Id) { $shellA } else { $shellB } $contender = if ($owner.Id -eq $shellA.Id) { $shellB } else { $shellA } - for ($i = 0; $i -lt 80; $i++) { - if ((Get-ShellSupervisorState $owner.Id) -eq 1 -and - (Get-ShellSupervisorState $contender.Id) -eq 2) { + $classificationDeadline = [DateTime]::UtcNow.AddSeconds(30) + while ([DateTime]::UtcNow -lt $classificationDeadline) { + Assert-ShellsAlive $shellA $shellB ( + Get-HandoffDiagnostics $shellA $shellB $daemonStateA $daemonStateB $handoffClock $daemonProcess $selectedChild + ) + $daemonProcess.Refresh() + if ($daemonProcess.HasExited) { + throw ( + "graphcoded exited before shell ownership classification completed: " + + (Get-HandoffDiagnostics $shellA $shellB $daemonStateA $daemonStateB $handoffClock $daemonProcess $selectedChild) + ) + } + $ownerState = Get-ShellSupervisorState $owner.Id + $contenderState = Get-ShellSupervisorState $contender.Id + if ($ownerState -eq 1 -and $contenderState -eq 2) { break } Start-Sleep -Milliseconds 100 @@ -216,9 +289,9 @@ try { $contenderState = Get-ShellSupervisorState $contender.Id if ($ownerState -ne 1 -or $contenderState -ne 2) { throw ( - "shell ownership classification was incorrect: owner=$ownerState, contender=$contenderState, " + - "daemonA=$(if (Test-Path $daemonStateA) { Get-Content $daemonStateA -Raw } else { '' }), " + - "daemonB=$(if (Test-Path $daemonStateB) { Get-Content $daemonStateB -Raw } else { '' })" + "shell ownership classification did not reach owner=1 and contender=2 before the deadline: " + + "owner=$ownerState, contender=$contenderState; " + + (Get-HandoffDiagnostics $shellA $shellB $daemonStateA $daemonStateB $handoffClock $daemonProcess $selectedChild) ) } Assert-EndpointReachable diff --git a/Tools/windows/Tests/TrayDaemon.Tests.ps1 b/Tools/windows/Tests/TrayDaemon.Tests.ps1 index 9524f9d4..deb68d4e 100644 --- a/Tools/windows/Tests/TrayDaemon.Tests.ps1 +++ b/Tools/windows/Tests/TrayDaemon.Tests.ps1 @@ -61,11 +61,21 @@ if ($daemonMain -notmatch "DaemonStartupHandoff" -or throw "child must skip the parent-held reservation and publish readiness after its lifetime lock" } $handoffLive = Join-Path $root "Tools\windows\Tests\DaemonHandoff.Live.Tests.ps1" -if (-not (Test-Path -LiteralPath $handoffLive) -or - (Get-Content -LiteralPath $handoffLive -Raw) -notmatch - "Concurrent shells did not spawn exactly one graphcoded child") { +if (-not (Test-Path -LiteralPath $handoffLive)) { throw "concurrent two-shell handoff coverage is missing" } +$handoffLiveText = Get-Content -LiteralPath $handoffLive -Raw +if ($handoffLiveText -notmatch "Concurrent shells did not spawn exactly one graphcoded child") { + throw "concurrent two-shell handoff coverage is missing" +} +if ($handoffLiveText -notmatch "classificationDeadline" -or + $handoffLiveText -notmatch "Assert-ShellsAlive" -or + $handoffLiveText -notmatch "Get-HandoffDiagnostics") { + throw "concurrent handoff test must use a diagnostic condition wait for ownership classification" +} +if ($handoffLiveText -match 'for \(\$i = 0; \$i -lt 80; \$i\+\+\) \{[\s\S]{0,400}Get-ShellSupervisorState \$owner\.Id') { + throw "concurrent handoff test must not regress to short fixed ownership polling" +} if ($app -notmatch "GRAPHCODE_DAEMON_SUPERVISOR_TEST_HOOK" -or $app -notmatch "DaemonSupervisorState") { throw "concurrent handoff test observability is missing"