Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 132 additions & 0 deletions .github/workflows/test-windows-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
name: Test Published Windows Release

on:
pull_request:
paths:
- '.github/workflows/test-windows-release.yml'
- 'scripts/test-windows-release.ps1'
workflow_dispatch:
inputs:
version:
description: 'Existing release version to test (no release is created)'
required: true
default: '1.0.77'

permissions:
contents: read

jobs:
windows-release:
name: Windows ${{ matrix.architecture }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
include:
- architecture: x64
runner: windows-2022
- architecture: arm64
runner: windows-11-arm
defaults:
run:
shell: pwsh
env:
RELEASE_VERSION: ${{ inputs.version || '1.0.77' }}
RELEASE_ARCHITECTURE: ${{ matrix.architecture }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
persist-credentials: false

- name: Download existing release assets
env:
GH_TOKEN: ${{ github.token }}
run: |
if ($env:RELEASE_VERSION -notmatch '^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$') {
throw 'Invalid release version'
}
$assets = Join-Path $env:GITHUB_WORKSPACE '.agent-work/windows-release/assets'
New-Item -ItemType Directory -Path $assets -Force | Out-Null
$prefix = "modernize_$($env:RELEASE_VERSION)_windows_$($env:RELEASE_ARCHITECTURE)"
gh release download "v$env:RELEASE_VERSION" --repo microsoft/modernize-cli `
--pattern "$prefix.msi" --pattern "$prefix.zip" `
--pattern "modernize_$($env:RELEASE_VERSION)_checksums.txt" --dir $assets
if ($LASTEXITCODE -ne 0) { throw 'Release download failed' }

- name: Test signed artifacts and native startup without installing
run: |
./scripts/test-windows-release.ps1 -Version $env:RELEASE_VERSION `
-Architecture $env:RELEASE_ARCHITECTURE `
-AssetDirectory .agent-work/windows-release/assets `
-WorkDirectory .agent-work/windows-release/work `
-OutputDirectory .agent-work/windows-release/results

- name: Approximate WinGet installation validation
run: |
./scripts/test-windows-release.ps1 -Mode Installation `
-Version $env:RELEASE_VERSION `
-Architecture $env:RELEASE_ARCHITECTURE `
-AssetDirectory .agent-work/windows-release/assets `
-WorkDirectory .agent-work/windows-release/installation-work `
-OutputDirectory .agent-work/windows-release/results

- name: Test MSI and missing-runtime baseline in disposable Windows container
if: matrix.architecture == 'x64'
run: |
$assets = Join-Path $env:GITHUB_WORKSPACE '.agent-work/windows-release/assets'
$results = Join-Path $env:GITHUB_WORKSPACE '.agent-work/windows-release/results'
$scripts = Join-Path $env:GITHUB_WORKSPACE 'scripts'
$vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"
$visualStudio = & $vswhere -latest -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath
if ($LASTEXITCODE -ne 0 -or -not $visualStudio) { throw 'Visual Studio C++ tools unavailable' }
$redist = Get-ChildItem (Join-Path $visualStudio 'VC\Redist\MSVC') -Directory |
Where-Object Name -Match '^\d+\.\d+\.\d+$' |
Sort-Object { [version]$_.Name } -Descending | Select-Object -First 1
if (-not $redist) { throw 'Visual Studio redistributable directory unavailable' }
$crt = Join-Path $redist.FullName 'x64\Microsoft.VC143.CRT'
$appLocal = New-Item -ItemType Directory -Path (Join-Path $assets 'vc-runtime-x64')
$runtimeFiles = @(Get-Item (Join-Path $crt 'vcruntime140.dll'))
if ($runtimeFiles.Count -ne 1) { throw 'Official vcruntime140.dll unavailable' }
foreach ($runtimeFile in $runtimeFiles) {
$signature = Get-AuthenticodeSignature $runtimeFile.FullName
if ($signature.Status -ne 'Valid' -or $signature.SignerCertificate.Subject -notmatch 'O=Microsoft Corporation') {
throw "Invalid official runtime DLL: $($runtimeFile.Name)"
}
Copy-Item $runtimeFile.FullName -Destination $appLocal.FullName
}
$runtime = Join-Path $assets 'vc_redist.x64.exe'
Invoke-WebRequest https://aka.ms/vc14/vc_redist.x64.exe -OutFile $runtime
$signature = Get-AuthenticodeSignature $runtime
if ($signature.Status -ne 'Valid' -or $signature.SignerCertificate.Subject -notmatch 'O=Microsoft Corporation') {
throw 'Official VC runtime installer signature verification failed'
}
Get-FileHash $runtime -Algorithm SHA256 | Format-List
docker info --format '{{.OSType}}'
if ($LASTEXITCODE -ne 0) { throw 'Windows container daemon unavailable' }
$containerArguments = @(
'run', '--rm', '--isolation=process', '--network', 'none', '--memory', '4g', '--cpus', '2',
'--env', 'MODERNIZE_RELEASE_CONTAINER=1',
'--mount', "type=bind,source=$assets,target=C:\input,readonly",
'--mount', "type=bind,source=$scripts,target=C:\tests,readonly",
'--mount', "type=bind,source=$results,target=C:\results",
'mcr.microsoft.com/windows/servercore:ltsc2022',
'powershell.exe', '-NoLogo', '-NoProfile', '-ExecutionPolicy', 'Bypass',
'-File', 'C:\tests\test-windows-release.ps1', '-Mode', 'Container',
'-Version', $env:RELEASE_VERSION, '-Architecture', 'x64',
'-AssetDirectory', 'C:\input', '-WorkDirectory', 'C:\work', '-OutputDirectory', 'C:\results'
)
if ($env:RELEASE_VERSION -eq '1.0.77') {
$containerArguments += '-ValidateAppLocalRemediation'
}
& docker @containerArguments
if ($LASTEXITCODE -ne 0) { throw 'Clean-container release test failed; inspect Container-x64.json and MSI logs' }

- name: Upload diagnostic results
if: always()
uses: actions/upload-artifact@v4
with:
name: windows-release-${{ matrix.architecture }}
path: .agent-work/windows-release/results/
if-no-files-found: warn
retention-days: 7
257 changes: 257 additions & 0 deletions scripts/test-windows-release.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,257 @@
param(
[Parameter(Mandatory = $true)]
[ValidatePattern('^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$')]
[string]$Version,

[ValidateSet('x64', 'arm64')]
[string]$Architecture = 'x64',

[ValidateSet('Host', 'Container', 'Installation')]
[string]$Mode = 'Host',

[Parameter(Mandatory = $true)]
[string]$AssetDirectory,

[Parameter(Mandatory = $true)]
[string]$WorkDirectory,

[Parameter(Mandatory = $true)]
[string]$OutputDirectory,

[switch]$ValidateAppLocalRemediation
)

$ErrorActionPreference = 'Stop'
if ($Mode -eq 'Container' -and $env:MODERNIZE_RELEASE_CONTAINER -ne '1') {
throw 'Container mode installs software and must only be invoked by the isolated test workflow.'
}

foreach ($directory in @($WorkDirectory, $OutputDirectory)) {
[void](New-Item -ItemType Directory -Path $directory -Force)
}
$AssetDirectory = (Resolve-Path -LiteralPath $AssetDirectory).Path
$WorkDirectory = (Resolve-Path -LiteralPath $WorkDirectory).Path
$OutputDirectory = (Resolve-Path -LiteralPath $OutputDirectory).Path
$env:MODERNIZE_CONFIG_HOME = Join-Path $WorkDirectory 'config'
$report = [ordered]@{
Version = $Version
Architecture = $Architecture
Mode = $Mode
OSVersion = [Environment]::OSVersion.VersionString
SystemVcRuntimePresent = Test-Path "$env:WINDIR\System32\vcruntime140.dll"
Passed = $false
}

function Invoke-ReleaseProcess {
param(
[string]$Executable,
[string]$Arguments,
[int]$TimeoutSeconds = 30
)

$startInfo = New-Object Diagnostics.ProcessStartInfo
$startInfo.FileName = $Executable
$startInfo.Arguments = $Arguments
$startInfo.WorkingDirectory = Split-Path -Parent $Executable
$startInfo.UseShellExecute = $false
$startInfo.CreateNoWindow = $true
$startInfo.RedirectStandardOutput = $true
$startInfo.RedirectStandardError = $true
$startInfo.RedirectStandardInput = $true
foreach ($secret in @('GH_TOKEN', 'GITHUB_TOKEN', 'COPILOT_GITHUB_TOKEN')) {
$startInfo.EnvironmentVariables.Remove($secret)
}
$process = New-Object Diagnostics.Process
$process.StartInfo = $startInfo
try {
[void]$process.Start()
$process.StandardInput.Close()
$stdout = $process.StandardOutput.ReadToEndAsync()
$stderr = $process.StandardError.ReadToEndAsync()
$timedOut = -not $process.WaitForExit($TimeoutSeconds * 1000)
if ($timedOut) {
& "$env:WINDIR\System32\taskkill.exe" /PID $process.Id /T /F | Out-Null
if (-not $process.WaitForExit(10000)) {
throw "Cannot stop timed-out process: $Executable"
}
}
return [pscustomobject]@{
Executable = Split-Path -Leaf $Executable
Arguments = $Arguments
TimedOut = $timedOut
ExitCode = $process.ExitCode
Stdout = $stdout.GetAwaiter().GetResult()
Stderr = $stderr.GetAwaiter().GetResult()
}
} finally {
$process.Dispose()
}
}

function Assert-Probe {
param($Probe, [int[]]$ExitCodes = @(0), [string]$OutputPattern)

if ($Probe.TimedOut -or $Probe.ExitCode -notin $ExitCodes) {
throw "Startup probe failed: $($Probe | ConvertTo-Json -Compress)"
}
if ($OutputPattern -and "$($Probe.Stdout)$($Probe.Stderr)" -notmatch $OutputPattern) {
throw "Unexpected output: $($Probe | ConvertTo-Json -Compress)"
}
}

try {
$prefix = "modernize_${Version}_windows_${Architecture}"
$msi = Join-Path $AssetDirectory "$prefix.msi"
if ($Mode -eq 'Host') {
$expectedArchitecture = if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { 'arm64' } else { 'x64' }
if ($Architecture -ne $expectedArchitecture) {
throw "Native startup tests require $Architecture Windows; this process is $expectedArchitecture."
}
$checksums = @{}
foreach ($line in Get-Content (Join-Path $AssetDirectory "modernize_${Version}_checksums.txt")) {
if ($line -match '^([a-fA-F0-9]{64})\s+\*?(.+)$') {
$checksums[$Matches[2]] = $Matches[1]
}
}
$report.AssetHashes = @{}
foreach ($extension in @('msi', 'zip')) {
$name = "$prefix.$extension"
$hash = (Get-FileHash (Join-Path $AssetDirectory $name) -Algorithm SHA256).Hash
$report.AssetHashes[$name] = $hash
if (-not $checksums.ContainsKey($name) -or $hash -ne $checksums[$name]) {
throw "Release checksum mismatch: $name"
}
}
$payload = Join-Path $WorkDirectory 'payload'
Expand-Archive (Join-Path $AssetDirectory "$prefix.zip") $payload -Force
$plugin = Join-Path $payload "runtimes\win-$Architecture\native\plugins\computer-use"
$report.Signatures = @()
foreach ($binary in @($msi, (Join-Path $payload 'modernize.exe'),
(Join-Path $plugin 'computer-use-mcp.exe'), (Join-Path $plugin 'CopilotComputerUse.exe'))) {
$signature = Get-AuthenticodeSignature -LiteralPath $binary
$report.Signatures += [pscustomobject]@{
File = Split-Path -Leaf $binary
Status = $signature.Status.ToString()
Signer = $signature.SignerCertificate.Subject
}
if ($signature.Status -ne 'Valid') {
throw "Invalid signature: $binary ($($signature.Status))"
}
}
$report.BundledVcRuntime = @(Get-ChildItem $payload -Recurse -Filter 'vcruntime140*.dll' |
ForEach-Object { $_.FullName.Substring($payload.Length + 1) })
$report.CliVersion = Invoke-ReleaseProcess (Join-Path $payload 'modernize.exe') '--version'
Assert-Probe $report.CliVersion -OutputPattern ([regex]::Escape($Version))
$report.CliHelp = Invoke-ReleaseProcess (Join-Path $payload 'modernize.exe') '--help'
Assert-Probe $report.CliHelp -OutputPattern 'Usage'
$report.McpHelper = Invoke-ReleaseProcess (Join-Path $plugin 'computer-use-mcp.exe') '--help'
Assert-Probe $report.McpHelper -ExitCodes @(0, 2) -OutputPattern 'arguments are not supported|Usage'
$report.ComputerUseHelper = Invoke-ReleaseProcess (Join-Path $plugin 'CopilotComputerUse.exe') '--help'
Assert-Probe $report.ComputerUseHelper -OutputPattern 'Usage'
} elseif ($Mode -eq 'Installation') {
$installed = Join-Path $WorkDirectory 'installed'
$installLog = Join-Path $OutputDirectory 'msi-install.log'
$report.MsiInstall = Invoke-ReleaseProcess "$env:WINDIR\System32\msiexec.exe" `
"/i `"$msi`" /qn /norestart INSTALLFOLDER=`"$installed`" /l*v `"$installLog`"" 180
Assert-Probe $report.MsiInstall -ExitCodes @(0, 3010)
try {
$plugin = Join-Path $installed "runtimes\win-$Architecture\native\plugins\computer-use"
$report.CliInstalled = Invoke-ReleaseProcess (Join-Path $installed 'modernize.exe') '--version'
Assert-Probe $report.CliInstalled -OutputPattern ([regex]::Escape($Version))
$report.McpHelperInstalled = Invoke-ReleaseProcess (Join-Path $plugin 'computer-use-mcp.exe') '--help'
Assert-Probe $report.McpHelperInstalled -ExitCodes @(0, 2) -OutputPattern 'arguments are not supported|Usage'
$report.ComputerUseHelperInstalled = Invoke-ReleaseProcess (Join-Path $plugin 'CopilotComputerUse.exe') '--help'
Assert-Probe $report.ComputerUseHelperInstalled -OutputPattern 'Usage'

$defender = Get-MpComputerStatus
$report.DefenderStatus = [pscustomobject]@{
AntivirusEnabled = $defender.AntivirusEnabled
RealTimeProtectionEnabled = $defender.RealTimeProtectionEnabled
AntivirusSignatureVersion = $defender.AntivirusSignatureVersion
}
if (-not $defender.AntivirusEnabled) {
throw 'Microsoft Defender Antivirus is not enabled on the validation runner.'
}
$detectionsBefore = @((Get-MpThreatDetection | ConvertTo-Json -Compress))
Start-MpScan -ScanType CustomScan -ScanPath $installed
$detectionsAfter = @((Get-MpThreatDetection | ConvertTo-Json -Compress))
$report.NewDefenderDetections = @($detectionsAfter | Where-Object { $_ -notin $detectionsBefore })
if ($report.NewDefenderDetections.Count -ne 0) {
throw "Microsoft Defender detected installed candidate files: $($report.NewDefenderDetections -join ', ')"
}
} finally {
$uninstallLog = Join-Path $OutputDirectory 'msi-uninstall.log'
$report.MsiUninstall = Invoke-ReleaseProcess "$env:WINDIR\System32\msiexec.exe" `
"/x `"$msi`" /qn /norestart /l*v `"$uninstallLog`"" 180
Assert-Probe $report.MsiUninstall -ExitCodes @(0, 3010)
}
} else {
if ($Architecture -ne 'x64') {
throw 'The Windows Server Core container test currently supports x64 only.'
}
if ($report.SystemVcRuntimePresent) {
throw 'The container already has VC runtime installed; it cannot establish a clean baseline.'
}
Add-Type -TypeDefinition @'
using System.Runtime.InteropServices;
public static class ReleaseErrorMode {
[DllImport("kernel32.dll")] public static extern uint SetErrorMode(uint mode);
}
'@
[void][ReleaseErrorMode]::SetErrorMode(3)
$installed = Join-Path $WorkDirectory 'installed'
$installLog = Join-Path $OutputDirectory 'msi-install.log'
$report.MsiInstall = Invoke-ReleaseProcess "$env:WINDIR\System32\msiexec.exe" `
"/i `"$msi`" /qn /norestart INSTALLFOLDER=`"$installed`" /l*v `"$installLog`"" 180
Assert-Probe $report.MsiInstall -ExitCodes @(0, 3010)
$plugin = Join-Path $installed 'runtimes\win-x64\native\plugins\computer-use'
$helper = Join-Path $plugin 'computer-use-mcp.exe'
$report.CliBeforeRuntime = Invoke-ReleaseProcess (Join-Path $installed 'modernize.exe') '--version'
$report.HelperBeforeRuntime = Invoke-ReleaseProcess $helper '--help'
$runtimeFiles = @(Get-Item (Join-Path $AssetDirectory 'vc-runtime-x64\vcruntime140.dll'))
if ($runtimeFiles.Count -ne 1) {
throw 'App-local runtime comparison requires exactly one official vcruntime140.dll.'
}
$report.AppLocalRuntimeFiles = @($runtimeFiles | ForEach-Object {
[pscustomobject]@{ File = $_.Name; SHA256 = (Get-FileHash $_.FullName).Hash }
})
foreach ($runtimeFile in $runtimeFiles) {
Copy-Item $runtimeFile.FullName -Destination $plugin
}
$report.HelperWithAppLocalRuntime = Invoke-ReleaseProcess $helper '--help'
Assert-Probe $report.HelperWithAppLocalRuntime -ExitCodes @(0, 2) -OutputPattern 'arguments are not supported|Usage'
foreach ($runtimeFile in $runtimeFiles) {
Remove-Item (Join-Path $plugin $runtimeFile.Name)
}
$runtimeInstaller = Join-Path $AssetDirectory 'vc_redist.x64.exe'
$report.RuntimeInstall = Invoke-ReleaseProcess $runtimeInstaller `
"/install /quiet /norestart /log `"$(Join-Path $OutputDirectory 'vcredist-install.log')`"" 180
Assert-Probe $report.RuntimeInstall -ExitCodes @(0, 3010)
$report.CliAfterRuntime = Invoke-ReleaseProcess (Join-Path $installed 'modernize.exe') '--version'
Assert-Probe $report.CliAfterRuntime -OutputPattern ([regex]::Escape($Version))
$report.HelperAfterRuntime = Invoke-ReleaseProcess $helper '--help'
Assert-Probe $report.HelperAfterRuntime -ExitCodes @(0, 2) -OutputPattern 'arguments are not supported|Usage'
$report.MissingVcRuntimeReproduced = ($report.HelperBeforeRuntime.ExitCode -eq -1073741515)
$uninstallLog = Join-Path $OutputDirectory 'msi-uninstall.log'
$report.MsiUninstall = Invoke-ReleaseProcess "$env:WINDIR\System32\msiexec.exe" `
"/x `"$msi`" /qn /norestart /l*v `"$uninstallLog`"" 180
Assert-Probe $report.MsiUninstall -ExitCodes @(0, 3010)
if ($ValidateAppLocalRemediation -and -not $report.MissingVcRuntimeReproduced) {
throw 'Expected the unchanged release helper to reproduce the missing-runtime loader failure.'
}
if (-not $ValidateAppLocalRemediation -and $report.MissingVcRuntimeReproduced) {
throw 'Release helper requires an externally installed VC runtime: clean startup failed, then succeeded after installing VC runtime.'
}
Assert-Probe $report.CliBeforeRuntime -OutputPattern ([regex]::Escape($Version))
if (-not $ValidateAppLocalRemediation) {
Assert-Probe $report.HelperBeforeRuntime -ExitCodes @(0, 2) -OutputPattern 'arguments are not supported|Usage'
}
}
$report.Passed = $true
} catch {
$report.Error = $_.Exception.Message
throw
} finally {
$report | ConvertTo-Json -Depth 8 | Set-Content (Join-Path $OutputDirectory "$Mode-$Architecture.json") -Encoding UTF8
$report | ConvertTo-Json -Depth 8 | Write-Output
}
Loading