diff --git a/Modules/BackupEngine/Public/Invoke-SEBBackup.ps1 b/Modules/BackupEngine/Public/Invoke-SEBBackup.ps1 index 2f089cd..603c477 100644 --- a/Modules/BackupEngine/Public/Invoke-SEBBackup.ps1 +++ b/Modules/BackupEngine/Public/Invoke-SEBBackup.ps1 @@ -433,6 +433,8 @@ function Invoke-SEBBackup { # Prune unchanged files from staging on the node. NON-IDEMPOTENT (deletes files); # -RetryCount 0 so a transport drop after the prune does not re-run it. A failure # throws and aborts the backup (the archive would otherwise carry the wrong delta). + # | Out-Null: the wrapper returns the remote block's output; this block is side-effect + # only, so discard it so nothing leaks into Invoke-SEBBackup's own (single-object) output. Invoke-SEBRemoteCommand -Session $session -SessionRef ([ref]$session) -RetryCount 0 -ScriptBlock { param($StagingDir, $KeepRelative) $keep = [System.Collections.Generic.HashSet[string]]::new( @@ -445,7 +447,7 @@ function Invoke-SEBBackup { Remove-Item -LiteralPath $_.FullName -Force -ErrorAction SilentlyContinue } } - } -ArgumentList @($nodeStagingDir, $changedFiles) + } -ArgumentList @($nodeStagingDir, $changedFiles) | Out-Null } # ======================================================================== @@ -466,12 +468,15 @@ function Invoke-SEBBackup { # compression step (which surfaces its own error if the handle is truly dead). try { $session = New-SEBSession -NodeName $NodeName -NodeConfig @{ hostname = $nodeHostname } } catch { Write-Verbose "Cache refresh before compression failed; using existing session. $_" } + # | Out-Null: Compress-SEBArchive returns a summary PSCustomObject the orchestrator does not + # consume (the authoritative size is read back from the node below). Discard it so it does + # not leak into Invoke-SEBBackup's output stream and turn the caller's $result into an array. Compress-SEBArchive ` -SourcePath $nodeStagingDir ` -DestinationPath $nodeArchivePath ` -Session $session ` -Engine $resolvedEngine ` - -CompressionLevel $compressionLevel + -CompressionLevel $compressionLevel | Out-Null # Get archive size from node $archiveInfo = Invoke-SEBRemoteCommand -Session $session -SessionRef ([ref]$session) -ScriptBlock { @@ -530,11 +535,13 @@ function Invoke-SEBBackup { $netBandwidthMbps = if ($globalConfig.network.max_bandwidth_mbps) { [int]$globalConfig.network.max_bandwidth_mbps } else { 0 } $netRobocopyIpgMs = if ($globalConfig.network.robocopy_ipg_ms) { [int]$globalConfig.network.robocopy_ipg_ms } else { 0 } + # | Out-Null: Copy-SEBThrottled returns a transfer-result PSCustomObject the orchestrator + # does not consume; discard it so it does not leak into this function's single-object output. Copy-SEBThrottled ` -Source $shareArchivePath ` -Destination $ccArchivePath ` -MaxBandwidthMbps $netBandwidthMbps ` - -RobocopyIpgMs $netRobocopyIpgMs + -RobocopyIpgMs $netRobocopyIpgMs | Out-Null $result.ArchiveFile = $ccArchivePath @@ -662,11 +669,13 @@ function Invoke-SEBBackup { Write-SEBLog -Message "Copying archive to NAS: $nasArchivePath" -Level INFO -Context $InstanceName } + # | Out-Null: discard Copy-SEBThrottled's result object (not consumed) so it does not + # leak into Invoke-SEBBackup's output stream. Copy-SEBThrottled ` -Source $ccArchivePath ` -Destination $nasArchivePath ` -MaxBandwidthMbps $netBandwidthMbps ` - -RobocopyIpgMs $netRobocopyIpgMs + -RobocopyIpgMs $netRobocopyIpgMs | Out-Null if ($hasLogger) { Write-SEBLog -Message "NAS copy completed." -Level INFO -Context $InstanceName @@ -771,7 +780,7 @@ function Invoke-SEBBackup { } Remove-Item -Path $archive.FullName -Force -ErrorAction SilentlyContinue } - } -ArgumentList $nodeStagingDir, $nodeArchivePath -ErrorAction Stop + } -ArgumentList $nodeStagingDir, $nodeArchivePath -ErrorAction Stop | Out-Null } catch { $warnings.Add("Node staging cleanup failed: $_") diff --git a/Modules/RestoreEngine/Public/Invoke-SEBRestore.ps1 b/Modules/RestoreEngine/Public/Invoke-SEBRestore.ps1 index f06ce76..4144d1d 100644 --- a/Modules/RestoreEngine/Public/Invoke-SEBRestore.ps1 +++ b/Modules/RestoreEngine/Public/Invoke-SEBRestore.ps1 @@ -256,13 +256,16 @@ function Invoke-SEBRestore { # -RetryCount 0 so a transport drop mid-operation does not re-run it. A hard failure # throws and aborts before any archive work. Route through the wrapper for # logging/reconnect; the block stays node-local. + # | Out-Null on each side-effect-only remote block below: the wrapper returns the block's + # remote output, and none of these are consumed -- discard them so they never leak into + # Invoke-SEBRestore's own output stream (which must be the single result PSCustomObject). Invoke-SEBRemoteCommand -Session $session -SessionRef ([ref]$session) -RetryCount 0 -ScriptBlock { param($tempDir) if (Test-Path -Path $tempDir -PathType Container) { Remove-Item -Path $tempDir -Recurse -Force -ErrorAction Stop } New-Item -Path $tempDir -ItemType Directory -Force -ErrorAction Stop | Out-Null - } -ArgumentList $tempRestoreDir + } -ArgumentList $tempRestoreDir | Out-Null # Extract each archive in chain order for ($i = 0; $i -lt $chainValidation.ChainArchives.Count; $i++) { @@ -301,7 +304,9 @@ function Invoke-SEBRestore { $shareDestPath = Join-Path -Path $sharePath -ChildPath "restore_temp_$(Split-Path -Path $archivePath -Leaf)" $netBandwidthMbps = if ($globalConfig.network.max_bandwidth_mbps) { [int]$globalConfig.network.max_bandwidth_mbps } else { 0 } $netRobocopyIpgMs = if ($globalConfig.network.robocopy_ipg_ms) { [int]$globalConfig.network.robocopy_ipg_ms } else { 0 } - Copy-SEBThrottled -Source $archivePath -Destination $shareDestPath -MaxBandwidthMbps $netBandwidthMbps -RobocopyIpgMs $netRobocopyIpgMs + # | Out-Null: discard Copy-SEBThrottled's result object (not consumed) so it does not + # leak into Invoke-SEBRestore's output stream. + Copy-SEBThrottled -Source $archivePath -Destination $shareDestPath -MaxBandwidthMbps $netBandwidthMbps -RobocopyIpgMs $netRobocopyIpgMs | Out-Null # Get the local path on the node for the archive. # Raw Invoke-Command (not the wrapper): -EA SilentlyContinue here means "best effort, @@ -377,7 +382,7 @@ function Invoke-SEBRestore { Remove-Item -Path $incTempDir -Recurse -Force -ErrorAction SilentlyContinue } } - } -ArgumentList $nodeArchiveTempPath, $tempRestoreDir, $archiveType + } -ArgumentList $nodeArchiveTempPath, $tempRestoreDir, $archiveType | Out-Null # Process deleted_files for incrementals if ($archiveType -eq 'incremental' -and $archiveManifest.ContainsKey('deleted_files')) { @@ -394,7 +399,7 @@ function Invoke-SEBRestore { Remove-Item -Path $fullPath -Force -ErrorAction SilentlyContinue } } - } -ArgumentList $tempRestoreDir, $deletedFiles -ErrorAction SilentlyContinue + } -ArgumentList $tempRestoreDir, $deletedFiles -ErrorAction SilentlyContinue | Out-Null if ($hasLogger) { Write-SEBLog -Message "Processed $($deletedFiles.Count) deleted file(s) for sequence $archiveSeq." -Level INFO -Context $InstanceName @@ -410,7 +415,7 @@ function Invoke-SEBRestore { if (Test-Path -Path $archPath -PathType Leaf) { Remove-Item -Path $archPath -Force -ErrorAction SilentlyContinue } - } -ArgumentList $nodeArchiveTempPath -ErrorAction SilentlyContinue + } -ArgumentList $nodeArchiveTempPath -ErrorAction SilentlyContinue | Out-Null } # ==================================================================== @@ -581,7 +586,7 @@ function Invoke-SEBRestore { (Get-ChildItem -Path $parentDir -ErrorAction SilentlyContinue).Count -eq 0) { Remove-Item -Path $parentDir -Force -ErrorAction SilentlyContinue } - } -ArgumentList $tempRestoreDir -ErrorAction Stop + } -ArgumentList $tempRestoreDir -ErrorAction Stop | Out-Null } catch { $warnings.Add("Cleanup of temp restore directory failed: $_") diff --git a/Tests/BackupEngine/Invoke-SEBBackup.Tests.ps1 b/Tests/BackupEngine/Invoke-SEBBackup.Tests.ps1 new file mode 100644 index 0000000..d4213b0 --- /dev/null +++ b/Tests/BackupEngine/Invoke-SEBBackup.Tests.ps1 @@ -0,0 +1,840 @@ +#Requires -Module Pester + +# Invoke-SEBBackup is the backup orchestrator -- the DATA PATH. It is responsible for sequencing +# the whole pipeline (configs -> session -> lock -> load check -> preflight -> VRage save -> VSS +# capture -> manifest -> compress -> transfer -> integrity -> NAS -> metrics -> notify -> retention +# -> staging cleanup) and, critically, for ALWAYS releasing the per-instance lock and tearing down +# the node PSSession in its finally block no matter which step failed. These tests mock the infra +# boundary (the SEB seam functions) and assert REAL orchestration behavior: that the pipeline runs +# in order on the happy path, that a failure in any one seam aborts correctly without reporting +# partial success, and that the lock/session are released on every path. +# +# Test notes: +# * Mocks target -ModuleName BackupEngine because Invoke-SEBBackup runs in that module's scope. +# Seams that live OUTSIDE BackupEngine (New-SEBSession, Invoke-SEBWithShadowCopy, New-SEBManifest, +# Compress-SEBArchive, Copy-SEBThrottled, Send-SEBBackupNotification, ...) are imported into that +# scope at module load, so mocking them -ModuleName BackupEngine intercepts the engine's calls. +# Seams INSIDE BackupEngine (Test-SEBPreFlight, Get-SEBBackupType, New-SEBLockFile, +# Remove-SEBExpiredBackups) are mocked the same way. +# * Each It calls Invoke-SEBBackup then asserts in the SAME block: Pester 5 scopes a mock's call +# history to the block that produced it, so invoke-in-BeforeAll / assert-in-It records zero calls. +# * cc_backup_root / nas_backup_path point at real temp dirs because the orchestrator does real +# New-Item / Test-Path / Rename-Item on the C&C side (only the NODE side is mocked). +# * New-FakeSession returns a type-satisfying but uninitialized PSSession (no transport); every +# consumer of it is mocked, so its internals are never touched. + +BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../..").Path + Import-Module "$repoRoot/SEBackup.psd1" -Force -DisableNameChecking 3>$null + . "$repoRoot/Tests/_TestHelpers/Test-Doubles.ps1" + + # A C&C temp root for this file's on-disk side effects (transfer dest, manifest write, NAS). + $script:ccRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("sebbk_cc_" + [guid]::NewGuid().ToString('n')) + $script:nasRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("sebbk_nas_" + [guid]::NewGuid().ToString('n')) + New-Item -Path $script:ccRoot, $script:nasRoot -ItemType Directory -Force | Out-Null + + # Build the global-config hashtable the orchestrator reads many nested keys from. Returned by the + # Get-SEBGlobalConfig mock. notifications.enabled/on_failure drive the notification assertions. + function New-GlobalConfig { + param([bool]$NotificationsEnabled = $true, [bool]$OnFailure = $true, [string]$Nas = $null) + @{ + load_awareness = @{ enabled = $false } + storage = @{ cc_backup_root = $script:ccRoot; nas_backup_path = $Nas } + compression = @{ engine = 'dotnet'; level_7zip = 5 } + network = @{ max_bandwidth_mbps = 0; robocopy_ipg_ms = 0 } + notifications = @{ enabled = $NotificationsEnabled; on_failure = $OnFailure } + defaults = @{ + vrage_api = @{ port = 8080; save_timeout_seconds = 60 } + vss = @{ mount_base = 'C:\SEBackup\vss_mount' } + } + } + } + + # A v2-shaped manifest hashtable, as New-SEBManifest returns it. The orchestrator mutates + # archive_path/_archive_path and reads files/chain_id/chain_sequence. + function New-ManifestObj { + param([string]$Type = 'full', [string]$ChainId = 'chain-aaa', [int]$Seq = 0) + @{ + version = 2 + type = $Type + chain_id = $ChainId + chain_sequence = $Seq + timestamp = [datetime]::UtcNow.ToString('o') + files = @{ 'Sandbox.sbc' = @{ size = 10; sha256 = ('a' * 64); last_write = [datetime]::UtcNow.ToString('o') } } + deleted_files = @() + } + } + + # Install the full set of "everything succeeds" mocks. Individual Contexts re-Mock one seam to + # fail. $BackupTypeObj lets a Context choose full vs incremental decision output. + function Set-HappyMocks { + param( + [hashtable]$GlobalConfig, + [PSCustomObject]$BackupTypeObj, + [hashtable]$ManifestObj + ) + + Mock Write-SEBLog {} -ModuleName BackupEngine + + Mock Get-SEBGlobalConfig -ModuleName BackupEngine { $GlobalConfig }.GetNewClosure() + Mock Get-SEBNodeConfig -ModuleName BackupEngine { @{ node = @{ hostname = 'node01' } } } + Mock New-SEBSession -ModuleName BackupEngine { New-FakeSession -Name 'SEBackup-node01' -Target 'node01' } + Mock Remove-SEBSession -ModuleName BackupEngine {} + Mock Get-SEBInstanceConfig -ModuleName BackupEngine { + @{ + world_path = 'D:\Torch\Instance\Saves\MyWorld' + staging_path = 'C:\SEBackup\staging' + share_name = 'SEBackup$' + vrage_api = @{ port = 8080; security_key = 'k' } + } + } + + Mock New-SEBLockFile -ModuleName BackupEngine { + [PSCustomObject]@{ Acquired = $true; LockFilePath = 'X:\lock'; Reason = $null; StaleLockBroken = $false } + } + Mock Remove-SEBLockFile -ModuleName BackupEngine { $true } + + Mock Get-SEBBackupType -ModuleName BackupEngine { $BackupTypeObj }.GetNewClosure() + + Mock Test-SEBPreFlight -ModuleName BackupEngine { + [PSCustomObject]@{ Passed = $true; Failures = @(); Warnings = @() } + } + + Mock Save-SEBVRageWorld -ModuleName BackupEngine { + [PSCustomObject]@{ Success = $true; Duration = [timespan]::FromSeconds(1); ErrorMessage = $null } + } + + Mock Get-SEBCompressionEngine -ModuleName BackupEngine { [PSCustomObject]@{ Engine = 'dotnet'; Version = '1'; Path = $null } } + + # VSS capture (node-local copy into staging). Returns nothing; just needs to not throw. + Mock Invoke-SEBWithShadowCopy -ModuleName BackupEngine {} + + # New-SEBManifest produces the manifest the rest of the pipeline carries. + Mock New-SEBManifest -ModuleName BackupEngine { $ManifestObj }.GetNewClosure() + Mock Compare-SEBManifest -ModuleName BackupEngine { + # Only called for incrementals. One changed file so the prune branch runs with content. + @{ Added = @('Sandbox.sbc'); Modified = @(); Deleted = @(); AddedCount = 1; ModifiedCount = 0; DeletedCount = 0; UnchangedCount = 0 } + } + + # Return the REAL .OUTPUTS shapes the production seams emit (not a fictional @{Success=$true}), + # so the single-object contract assertions actually exercise the orchestrator's `| Out-Null` + # discards: if an Out-Null is removed, the (now realistic) object leaks into the output stream + # and @($r).Count -ne 1 (mutation-checked). + # Compress-SEBArchive -> @{ ArchivePath; SizeBytes; Engine; Duration } + Mock Compress-SEBArchive -ModuleName BackupEngine { + [PSCustomObject]@{ ArchivePath = $DestinationPath; SizeBytes = 12345; Engine = 'dotnet'; Duration = [timespan]::FromSeconds(2) } + } + Mock Get-SEBSharePath -ModuleName BackupEngine { '\\node01\SEBackup$' } + # Copy-SEBThrottled -> @{ Source; Destination; SizeBytes; DurationSeconds; AverageMbps; Method } + Mock Copy-SEBThrottled -ModuleName BackupEngine { + [PSCustomObject]@{ Source = $Source; Destination = $Destination; SizeBytes = 12345; DurationSeconds = 1.0; AverageMbps = 98.7; Method = 'Robocopy' } + } + Mock Write-SEBManifest -ModuleName BackupEngine {} + + # Real .OUTPUTS shapes (orchestrator reads .Passed / .ErrorMessage): + # Test-SEBArchiveIntegrity -> @{ Level; Passed; ArchivePath; CheckedAt; ErrorMessage } + # Test-SEBManifestIntegrity -> @{ Level; Passed; ArchivePath; ManifestPath; FileCountMatch; HashMatch; Mismatches; CheckedAt; ErrorMessage } + Mock Test-SEBArchiveIntegrity -ModuleName BackupEngine { + [PSCustomObject]@{ Level = 1; Passed = $true; ArchivePath = $ArchivePath; CheckedAt = [datetime]::UtcNow; ErrorMessage = $null } + } + Mock Test-SEBManifestIntegrity -ModuleName BackupEngine { + [PSCustomObject]@{ Level = 2; Passed = $true; ArchivePath = $ArchivePath; ManifestPath = $ManifestPath; FileCountMatch = $true; HashMatch = $true; Mismatches = @(); CheckedAt = [datetime]::UtcNow; ErrorMessage = $null } + } + Mock Add-SEBMetric -ModuleName BackupEngine {} + Mock Send-SEBBackupNotification -ModuleName BackupEngine {} + Mock Remove-SEBExpiredBackups -ModuleName BackupEngine {} + + # Invoke-SEBRemoteCommand is called several times with DIFFERENT script blocks. Route by the + # block text so each call gets the shape the orchestrator expects to read back: + # * pathInfo (Split-Path -Qualifier) -> @{ VolumeRoot; WorldRelative } + # * archiveInfo (Get-Item .Length) -> @{ SizeBytes; Exists } + # * prune / staging-cleanup -> $null (side-effect only on the node) + # + # FRAGILITY (intentional, documented): these ParameterFilters disambiguate the remote calls by + # matching INCIDENTAL literals in each block's source text ('Split-Path .*-Qualifier' in the + # path-probe block; 'SizeBytes' in the archive-size block). Those literals are the only stable + # markers available without re-architecting the orchestrator to tag its remote blocks, but they + # are NOT a contract: if Invoke-SEBBackup.ps1 rewrites either block so the literal disappears + # (e.g. computes the qualifier differently, or renames the size field), the matching filter + # silently stops matching and that call falls through to the catch-all `$null` mock -- the + # backup then reads a $null pathInfo/archiveInfo and fails in a confusing way. If you touch + # those remote blocks, update the markers here in lock-step. The catch-all below is keyed as + # the NEGATION of the two specific markers for the same reason. + Mock Invoke-SEBRemoteCommand -ModuleName BackupEngine -ParameterFilter { $ScriptBlock.ToString() -match 'Split-Path .*-Qualifier' } { + @{ VolumeRoot = 'D:\'; WorldRelative = 'Torch\Instance\Saves\MyWorld' } + } + Mock Invoke-SEBRemoteCommand -ModuleName BackupEngine -ParameterFilter { $ScriptBlock.ToString() -match 'SizeBytes' } { + @{ SizeBytes = 12345; Exists = $true } + } + Mock Invoke-SEBRemoteCommand -ModuleName BackupEngine -ParameterFilter { + $ScriptBlock.ToString() -notmatch 'Split-Path .*-Qualifier' -and $ScriptBlock.ToString() -notmatch 'SizeBytes' + } { $null } + } +} + +AfterAll { + Remove-Item -LiteralPath $script:ccRoot -Recurse -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $script:nasRoot -Recurse -Force -ErrorAction SilentlyContinue +} + +# =========================================================================================== +# HAPPY PATHS +# =========================================================================================== +Describe 'Invoke-SEBBackup happy path (FULL)' { + BeforeAll { + $cfg = New-GlobalConfig -Nas $script:nasRoot + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'forced'; LastFullManifest = $null; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj -Type 'full' -ChainId 'chain-aaa' -Seq 0) + } + + It 'returns Success with the populated result object (archive/manifest/chain fields)' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull + + # The function's documented .OUTPUTS is a SINGLE PSCustomObject. Assert it is not an array: + # the data-path seams (Compress-SEBArchive, Copy-SEBThrottled, the side-effect remote blocks) + # return objects that, if left uncaptured, leak into this output stream and make the caller's + # $result an array -- silently breaking $result.Success for every caller. Pin the single-object + # contract here so that regression cannot return. + @($r).Count | Should -Be 1 + $r | Should -BeOfType ([System.Management.Automation.PSCustomObject]) + $r.Success | Should -BeTrue + $r.BackupType | Should -Be 'full' + $r.InstanceName | Should -Be 'PvPArena' + $r.NodeName | Should -Be 'node01' + $r.ArchiveFile | Should -Not -BeNullOrEmpty + $r.ManifestFile | Should -Not -BeNullOrEmpty + $r.ArchiveSizeBytes | Should -Be 12345 + $r.FileCount | Should -Be 1 + $r.ChainId | Should -Be 'chain-aaa' + $r.ChainSequence | Should -Be 0 + $r.IntegrityPassed | Should -BeTrue + $r.ErrorMessage | Should -BeNullOrEmpty + $r.Duration | Should -Not -BeNullOrEmpty + } + + It 'invokes each pipeline stage exactly once (lock, preflight, VSS, compress, manifest write, transfer, integrity, retention, notify)' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull | Out-Null + + # Each stage actually ran -- the assertions below are the "real orchestration" proof that the + # happy-path is not vacuous: if the orchestrator short-circuited, these would be 0. (This test + # asserts COUNTS only; the order-sensitive pairs are pinned in the next test.) + Should -Invoke New-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly -ParameterFilter { $InstanceName -eq 'PvPArena' } + Should -Invoke Test-SEBPreFlight -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Invoke-SEBWithShadowCopy -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke New-SEBManifest -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Compress-SEBArchive -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Write-SEBManifest -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Copy-SEBThrottled -ModuleName BackupEngine -Times 2 -Exactly # C&C transfer + NAS copy + Should -Invoke Test-SEBArchiveIntegrity -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Remove-SEBExpiredBackups -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Send-SEBBackupNotification -ModuleName BackupEngine -Times 1 -Exactly + } + + It 'orders the data path correctly: compress BEFORE the manifest write, and integrity BEFORE the NAS publish' { + # Counts alone cannot catch a reordering that, say, published to NAS before integrity ran. + # Record the ACTUAL call order: re-mock the ordered seams (on top of the happy mocks) so each + # appends its stage name to a script-scoped list, then assert the order-sensitive pairs. Each + # recorder still returns the real .OUTPUTS shape so the orchestrator runs through unchanged. + $script:order = [System.Collections.Generic.List[string]]::new() + Mock Invoke-SEBWithShadowCopy -ModuleName BackupEngine { $script:order.Add('vss') } + Mock Compress-SEBArchive -ModuleName BackupEngine { + $script:order.Add('compress') + [PSCustomObject]@{ ArchivePath = $DestinationPath; SizeBytes = 12345; Engine = 'dotnet'; Duration = [timespan]::FromSeconds(1) } + } + Mock Write-SEBManifest -ModuleName BackupEngine { $script:order.Add('manifest_write') } + Mock Test-SEBArchiveIntegrity -ModuleName BackupEngine { + $script:order.Add('integrity') + [PSCustomObject]@{ Level = 1; Passed = $true; ArchivePath = $ArchivePath; CheckedAt = [datetime]::UtcNow; ErrorMessage = $null } + } + Mock Copy-SEBThrottled -ModuleName BackupEngine { + # Tag the NAS copy distinctly from the C&C transfer so the integrity aborts BEFORE acquiring infra-heavy work; lock/session still released' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock Test-SEBPreFlight -ModuleName BackupEngine { + [PSCustomObject]@{ Passed = $false; Failures = @('insufficient disk space on node'); Warnings = @() } + } + } + + It 'returns failure with the preflight reason and never runs VSS/compress/manifest' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull + + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'Pre-flight' + $r.ErrorMessage | Should -Match 'insufficient disk space' + + Should -Invoke Invoke-SEBWithShadowCopy -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Compress-SEBArchive -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke New-SEBManifest -ModuleName BackupEngine -Times 0 -Exactly + } + + It 'still releases the lock and tears down the session (finally runs)' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly + } + } + + Context 'lock already held -> aborts with a clear result and does NOT run the backup' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock New-SEBLockFile -ModuleName BackupEngine { + [PSCustomObject]@{ Acquired = $false; LockFilePath = 'X:\lock'; Reason = 'a backup is already running'; StaleLockBroken = $false } + } + } + + It 'returns failure mentioning the lock and runs none of the data-path seams' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull + + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'lock' + $r.ErrorMessage | Should -Match 'already running' + + Should -Invoke Test-SEBPreFlight -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Invoke-SEBWithShadowCopy -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Compress-SEBArchive -ModuleName BackupEngine -Times 0 -Exactly + } + + It 'does NOT release a lock it never acquired (lockAcquired stays $false)' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull | Out-Null + # The lock was never ours, so finally must not call Remove-SEBLockFile. + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 0 -Exactly + # But the session WAS created before the lock check, so it is still torn down. + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly + } + } + + Context 'VSS capture fails -> aborts; no compress/manifest; lock released + session torn down in finally' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock Invoke-SEBWithShadowCopy -ModuleName BackupEngine { throw 'VSS snapshot creation failed on node' } + } + + It 'returns failure with the VSS error and skips compress + manifest generation' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull + + $r.Success | Should -BeFalse + $r.IntegrityPassed | Should -BeFalse + $r.ErrorMessage | Should -Match 'VSS' + + Should -Invoke New-SEBManifest -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Compress-SEBArchive -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Write-SEBManifest -ModuleName BackupEngine -Times 0 -Exactly + } + + It 'releases the lock and tears down the session even though VSS threw (finally)' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly -ParameterFilter { $InstanceName -eq 'PvPArena' } + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly -ParameterFilter { $NodeName -eq 'node01' } + } + } + + Context 'compress fails -> aborts; no manifest write/transfer; lock + session released' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock Compress-SEBArchive -ModuleName BackupEngine { throw 'compression engine returned a nonzero exit code' } + } + + It 'returns failure and never writes the manifest or transfers the archive' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'compression' + # Manifest WAS generated (before compress) but never written/transferred after the failure. + Should -Invoke Write-SEBManifest -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Copy-SEBThrottled -ModuleName BackupEngine -Times 0 -Exactly + } + + It 'releases the lock and tears down the session after a compress failure' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly + } + } + + Context 'archive missing after compress (Exists=$false) -> aborts before transfer' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + # archiveInfo reports the archive does not exist on the node. + Mock Invoke-SEBRemoteCommand -ModuleName BackupEngine -ParameterFilter { $ScriptBlock.ToString() -match 'SizeBytes' } { + @{ SizeBytes = 0; Exists = $false } + } + } + + It 'throws "Archive was not created" and does not transfer or write the manifest' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'Archive was not created' + Should -Invoke Copy-SEBThrottled -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Write-SEBManifest -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly + } + } + + Context 'transfer fails -> aborts; lock + session released' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + # The FIRST Copy-SEBThrottled is the C&C transfer; make it throw. + Mock Copy-SEBThrottled -ModuleName BackupEngine { throw 'robocopy failed transferring archive from share' } + } + + It 'returns failure from the transfer and never writes the C&C manifest' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'robocopy|transfer' + Should -Invoke Write-SEBManifest -ModuleName BackupEngine -Times 0 -Exactly + } + + It 'releases the lock and tears down the session after a transfer failure' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly + } + } + + Context 'manifest generation fails (New-SEBManifest returns $null) -> aborts before compress' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock New-SEBManifest -ModuleName BackupEngine { $null } + } + + It 'throws "Manifest generation failed" and never compresses' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'Manifest generation failed' + Should -Invoke Compress-SEBArchive -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly + } + } + + Context 'integrity fails -> NOT a hard error, but Success=$false, archive renamed _BAD, NO NAS publish' { + BeforeAll { + $cfg = New-GlobalConfig -Nas $script:nasRoot + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = 'chain-int'; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj -ChainId 'chain-int') + # The _BAD rename operates on the REAL on-disk C&C archive, so the C&C transfer mock must + # actually create that file for the rename to have something to act on (Copy-SEBThrottled + # is otherwise a no-op). The orchestrator transfers TO $ccArchivePath; create it there. + Mock Copy-SEBThrottled -ModuleName BackupEngine { + if ($Destination -and (Split-Path $Destination -Parent | Test-Path)) { + Set-Content -LiteralPath $Destination -Value 'corrupt-archive-bytes' -NoNewline -ErrorAction SilentlyContinue + } + [PSCustomObject]@{ Source = $Source; Destination = $Destination; SizeBytes = 21; DurationSeconds = 0.1; AverageMbps = 1.0; Method = 'Robocopy' } + } + # Level-1 integrity fails. The orchestrator does NOT throw -- it marks the backup BAD, + # renames archive+manifest, suppresses the NAS publish, and reports Success=$false. + # Real .OUTPUTS: @{ Level; Passed; ArchivePath; CheckedAt; ErrorMessage }. + Mock Test-SEBArchiveIntegrity -ModuleName BackupEngine { + [PSCustomObject]@{ Level = 1; Passed = $false; ArchivePath = $ArchivePath; CheckedAt = [datetime]::UtcNow; ErrorMessage = 'CRC mismatch' } + } + } + + # NOTE: each It uses a UNIQUE instance name. The orchestrator writes a REAL archive to + # {cc}\{Instance}\full\{Instance}_FULL_{yyyyMMdd_HHmmss}.zip and (on integrity failure) + # renames it to _BAD. Two calls in the same wall-clock second with the same instance name + # would produce the same path and the second _BAD rename would collide with the first run's + # _BAD file. Distinct instance names keep each It's on-disk side effects independent. + It 'reports IntegrityPassed=$false and Success=$false (no false-positive success)' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'IntgA' -ForceFull + $r.IntegrityPassed | Should -BeFalse + $r.Success | Should -BeFalse + } + + It 'renames the produced archive to a _BAD name on the result' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'IntgB' -ForceFull + # The real archive was written and renamed; the result must point at the _BAD artifact so + # retention/restore never treat the failed backup as a valid chain parent. + $r.ArchiveFile | Should -Match '_BAD' + Test-Path -LiteralPath $r.ArchiveFile | Should -BeTrue + } + + It 'does NOT copy the failed archive to NAS (integrity gate before NAS publish)' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'IntgC' -ForceFull | Out-Null + # Only the single C&C transfer should have happened -- the NAS Copy is gated on integrity. + Should -Invoke Copy-SEBThrottled -ModuleName BackupEngine -ParameterFilter { $Destination -like "$script:nasRoot*" } -Times 0 -Exactly + } + + It 'still releases the lock and session' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'IntgD' -ForceFull | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly + } + + It 'still runs retention on the integrity-failed path (a _BAD backup must not stop the sweep)' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'IntgRet' -ForceFull | Out-Null + # Integrity failure is the SOFT path (no throw): the orchestrator runs to completion through + # metrics/notify/retention, so the expired-backup sweep must still fire exactly once. + Should -Invoke Remove-SEBExpiredBackups -ModuleName BackupEngine -Times 1 -Exactly -ParameterFilter { $InstanceName -eq 'IntgRet' } + } + } + + Context 'integrity fails AND the _BAD rename itself fails -> partial state surfaced as a warning, not a crash' { + BeforeAll { + $cfg = New-GlobalConfig -Nas $script:nasRoot + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = 'chain-badren'; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj -ChainId 'chain-badren') + # The C&C transfer writes the REAL archive, then pre-creates the _BAD destination as a + # NON-EMPTY DIRECTORY at exactly the path the orchestrator will try to rename onto. A + # file->file Rename-Item -Force onto an existing directory of that name throws ("Cannot + # create a file when that file already exists"), exercising the catch that the happy _BAD + # rename never reaches. + Mock Copy-SEBThrottled -ModuleName BackupEngine { + if ($Destination -and (Split-Path $Destination -Parent | Test-Path)) { + Set-Content -LiteralPath $Destination -Value 'corrupt-archive-bytes' -NoNewline -ErrorAction SilentlyContinue + $badPath = $Destination -replace '(\.\w+)$', '_BAD$1' + New-Item -ItemType Directory -Path $badPath -Force -ErrorAction SilentlyContinue | Out-Null + Set-Content -LiteralPath (Join-Path $badPath 'blocker.txt') -Value 'x' -NoNewline -ErrorAction SilentlyContinue + } + [PSCustomObject]@{ Source = $Source; Destination = $Destination; SizeBytes = 21; DurationSeconds = 0.1; AverageMbps = 1.0; Method = 'Robocopy' } + } + Mock Test-SEBArchiveIntegrity -ModuleName BackupEngine { + [PSCustomObject]@{ Level = 1; Passed = $false; ArchivePath = $ArchivePath; CheckedAt = [datetime]::UtcNow; ErrorMessage = 'CRC mismatch' } + } + } + + It 'reports Success=$false, records the rename failure as a warning, and does NOT throw out of the function' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'IntgRenFail' -ForceFull + # The function must STILL return a structured result (the rename failure is caught, not + # propagated): integrity failed so Success/IntegrityPassed are false, and a warning records + # that the _BAD rename could not complete. + $r.Success | Should -BeFalse + $r.IntegrityPassed | Should -BeFalse + ($r.Warnings -join ' ') | Should -Match 'rename failed archive to BAD' + # Because the rename failed, ArchiveFile keeps the ORIGINAL (non-_BAD) path -- and that + # original file is still on disk (it was never moved). + $r.ArchiveFile | Should -Not -Match '_BAD' + Test-Path -LiteralPath $r.ArchiveFile | Should -BeTrue + } + + It 'still gates NAS publish off (a failed-integrity archive is never published even if its _BAD rename failed)' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'IntgRenFail2' -ForceFull | Out-Null + Should -Invoke Copy-SEBThrottled -ModuleName BackupEngine -ParameterFilter { $Destination -like "$script:nasRoot*" } -Times 0 -Exactly + } + + It 'still releases the lock and tears down the session on the partial-state path' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'IntgRenFail3' -ForceFull | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly + } + } +} + +# =========================================================================================== +# NOTIFICATION GATING +# =========================================================================================== +Describe 'Invoke-SEBBackup notification gating' { + + Context 'a hard failure with notifications.on_failure=$true sends a failure notification' { + BeforeAll { + $cfg = New-GlobalConfig -OnFailure $true + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock Compress-SEBArchive -ModuleName BackupEngine { throw 'boom' } + } + + It 'sends exactly one notification, carrying the failed result (Success=$false)' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull | Out-Null + Should -Invoke Send-SEBBackupNotification -ModuleName BackupEngine -Times 1 -Exactly -ParameterFilter { + $BackupResult.Success -eq $false + } + } + } + + Context 'a hard failure with notifications.on_failure=$false sends NO notification' { + BeforeAll { + $cfg = New-GlobalConfig -OnFailure $false + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock Compress-SEBArchive -ModuleName BackupEngine { throw 'boom' } + } + + It 'suppresses the failure notification per the on_failure gate' { + Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull | Out-Null + Should -Invoke Send-SEBBackupNotification -ModuleName BackupEngine -Times 0 -Exactly + } + } + + Context '-SkipNotify suppresses the notification on the success path' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + } + + It 'does not notify when -SkipNotify is set even though the backup succeeds' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull -SkipNotify + $r.Success | Should -BeTrue + Should -Invoke Send-SEBBackupNotification -ModuleName BackupEngine -Times 0 -Exactly + } + } +} + +# =========================================================================================== +# RE-ENTRANT CONTRACT (-SkipLock / -KeepSession) used by Invoke-SEBRestore's safety backup +# =========================================================================================== +Describe 'Invoke-SEBBackup re-entrant flags (-SkipLock / -KeepSession)' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + } + + It '-SkipLock does not acquire or release the per-instance lock (caller already holds it)' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull -SkipLock -SkipNotify + $r.Success | Should -BeTrue + Should -Invoke New-SEBLockFile -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 0 -Exactly + } + + It '-KeepSession does not tear down the node session (caller owns its lifecycle)' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull -KeepSession -SkipNotify + $r.Success | Should -BeTrue + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 0 -Exactly + } +} + +# =========================================================================================== +# LOAD-AWARENESS GATE + VRAGE BRANCHES +# =========================================================================================== +Describe 'Invoke-SEBBackup load-awareness gate' { + + Context 'node under high load, then waits and proceeds (records the delay as a warning)' { + BeforeAll { + # load_awareness.enabled = $true triggers the Test-SEBNodeLoad / Wait-SEBNodeLoad path. + $cfg = New-GlobalConfig + $cfg.load_awareness = @{ enabled = $true } + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock Test-SEBNodeLoad -ModuleName BackupEngine { [PSCustomObject]@{ CanProceed = $false } } + Mock Wait-SEBNodeLoad -ModuleName BackupEngine { [PSCustomObject]@{ CanProceed = $true; WaitedSeconds = 12 } } + } + + It 'consults Test-SEBNodeLoad, waits via Wait-SEBNodeLoad, then completes with a delay warning' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull -SkipNotify + $r.Success | Should -BeTrue + Should -Invoke Test-SEBNodeLoad -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Wait-SEBNodeLoad -ModuleName BackupEngine -Times 1 -Exactly + ($r.Warnings -join ' ') | Should -Match 'high node load' + } + } + + Context 'node never reaches safe load -> backup deferred (aborts) but lock/session released' { + BeforeAll { + $cfg = New-GlobalConfig + $cfg.load_awareness = @{ enabled = $true } + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock Test-SEBNodeLoad -ModuleName BackupEngine { [PSCustomObject]@{ CanProceed = $false } } + Mock Wait-SEBNodeLoad -ModuleName BackupEngine { [PSCustomObject]@{ CanProceed = $false; WaitedSeconds = 600 } } + } + + It 'aborts with a deferral message and never reaches VSS, releasing the lock and session' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull -SkipNotify + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'deferred' + Should -Invoke Invoke-SEBWithShadowCopy -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Remove-SEBLockFile -ModuleName BackupEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName BackupEngine -Times 1 -Exactly + } + } + + Context '-SkipLoadCheck bypasses the load gate even when load_awareness is enabled' { + BeforeAll { + $cfg = New-GlobalConfig + $cfg.load_awareness = @{ enabled = $true } + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock Test-SEBNodeLoad -ModuleName BackupEngine { [PSCustomObject]@{ CanProceed = $false } } + Mock Wait-SEBNodeLoad -ModuleName BackupEngine { [PSCustomObject]@{ CanProceed = $false; WaitedSeconds = 600 } } + } + + It 'never calls the load checks and completes the backup' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull -SkipLoadCheck -SkipNotify + $r.Success | Should -BeTrue + Should -Invoke Test-SEBNodeLoad -ModuleName BackupEngine -Times 0 -Exactly + Should -Invoke Wait-SEBNodeLoad -ModuleName BackupEngine -Times 0 -Exactly + } + } +} + +Describe 'Invoke-SEBBackup VRage save branch' { + + Context 'no VRage security_key configured -> warns and skips the world save, still succeeds' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + # Instance config WITHOUT a vrage_api.security_key -> the save-trigger branch is skipped. + Mock Get-SEBInstanceConfig -ModuleName BackupEngine { + @{ world_path = 'D:\Torch\Instance\Saves\MyWorld'; staging_path = 'C:\SEBackup\staging'; share_name = 'SEBackup$' } + } + } + + It 'does not call Save-SEBVRageWorld, adds a no-key warning, and still backs up' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull -SkipNotify + $r.Success | Should -BeTrue + Should -Invoke Save-SEBVRageWorld -ModuleName BackupEngine -Times 0 -Exactly + ($r.Warnings -join ' ') | Should -Match 'security_key' + } + } + + Context 'VRage save fails -> warning only, the backup continues (non-fatal)' { + BeforeAll { + $cfg = New-GlobalConfig + $bt = [PSCustomObject]@{ Type = 'full'; Reason = 'x'; LastManifest = $null; ChainId = $null; ChainSequence = 0 } + Set-HappyMocks -GlobalConfig $cfg -BackupTypeObj $bt -ManifestObj (New-ManifestObj) + Mock Save-SEBVRageWorld -ModuleName BackupEngine { + [PSCustomObject]@{ Success = $false; Duration = [timespan]::Zero; ErrorMessage = 'API connection refused' } + } + } + + It 'records the save failure as a warning but completes the backup successfully' { + $r = Invoke-SEBBackup -NodeName 'node01' -InstanceName 'PvPArena' -ForceFull -SkipNotify + $r.Success | Should -BeTrue + ($r.Warnings -join ' ') | Should -Match 'world save failed' + # The pipeline still proceeded past the save into the capture/compress steps. + Should -Invoke Invoke-SEBWithShadowCopy -ModuleName BackupEngine -Times 1 -Exactly + } + } +} + +# =========================================================================================== +# DRY-RUN CONTRACT: the engine orchestrator deliberately does NOT implement -WhatIf. +# =========================================================================================== +Describe 'Invoke-SEBBackup does not implement -WhatIf (dry-run lives at the Scripts layer)' { + # The engine is the data path; the -WhatIf/-Confirm risk gate is implemented by the entry-point + # script (Scripts/Invoke-Restore.ps1 has SupportsShouldProcess), NOT duplicated in the engine. + # This test pins that contract so a future half-wired WhatIf on the engine (which would guard + # only SOME mutating seams and silently no-op others) is caught in review. + It 'does not advertise SupportsShouldProcess (no -WhatIf parameter)' { + (Get-Command Invoke-SEBBackup).Parameters.ContainsKey('WhatIf') | Should -BeFalse + } +} diff --git a/Tests/RestoreEngine/Invoke-SEBRestore.Tests.ps1 b/Tests/RestoreEngine/Invoke-SEBRestore.Tests.ps1 new file mode 100644 index 0000000..845f0ab --- /dev/null +++ b/Tests/RestoreEngine/Invoke-SEBRestore.Tests.ps1 @@ -0,0 +1,426 @@ +#Requires -Module Pester + +# Invoke-SEBRestore is the restore orchestrator -- a DATA PATH that overwrites the live world. It +# sequences: lock -> configs/session -> chain validation -> SAFETY BACKUP -> reconstruct in temp -> +# VERIFY reconstruction -> stop server -> DEPLOY -> start server -> notify -> cleanup. The +# safety-critical invariants these tests assert (with the infra boundary mocked): +# * the per-instance lock is acquired up front and ALWAYS released in finally; +# * a chain-validation failure aborts BEFORE the safety backup or any node work; +# * a safety-backup failure aborts BEFORE the live world is touched (no stop/deploy); +# * the server is stopped only AFTER a verified reconstruction (deferred), and a deploy failure +# is surfaced as a structured failure (the rollback lives in Deploy-SEBRestoredFiles); +# * the canonical .ErrorMessage is populated on every failure and the result is a single object. +# +# Test notes: +# * Mocks target -ModuleName RestoreEngine (the orchestrator's scope). Seams imported into that +# scope (New-SEBSession, Stop/Start-SEBTorchServer, Deploy-SEBRestoredFiles, Invoke-SEBBackup, +# Copy-SEBThrottled, Send-SEBRestoreNotification, ...) are mocked there. +# * Test-SEBRestoreChain returns ChainManifests = REAL on-disk JSON paths, because the orchestrator +# reads each manifest with Get-Content | ConvertFrom-Json to drive deleted_files and verification. +# A temp manifests dir is created per Describe with engine-shaped v2 manifests. +# * Each It invokes then asserts in the same block (Pester 5 scopes mock call history per block). + +BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../..").Path + Import-Module "$repoRoot/SEBackup.psd1" -Force -DisableNameChecking 3>$null + . "$repoRoot/Tests/_TestHelpers/Test-Doubles.ps1" + + # A C&C backup root with a real manifests dir; Test-SEBRestoreChain mock points at these files. + $script:ccRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("sebrs_cc_" + [guid]::NewGuid().ToString('n')) + $script:manifestsDir = Join-Path (Join-Path $script:ccRoot 'PvPArena') 'manifests' + New-Item -Path $script:manifestsDir -ItemType Directory -Force | Out-Null + + function New-ChainManifestFile { + param([string]$Dir, [string]$Name, [string]$Type, [string]$ChainId, [int]$Seq, [string[]]$Deleted = @()) + $m = @{ + version = 2 + type = $Type + chain_id = $ChainId + chain_sequence = $Seq + parent_manifest = $(if ($Seq -gt 0) { 'parent.json' } else { $null }) + timestamp = [datetime]::UtcNow.ToString('o') + files = @{ 'Sandbox.sbc' = @{ size = 10; sha256 = ('a' * 64); last_write = [datetime]::UtcNow.ToString('o') } } + deleted_files = $Deleted + } + $path = Join-Path $Dir $Name + $m | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $path + return $path + } + + function New-RestoreGlobalConfig { + param([bool]$NotificationsEnabled = $true) + @{ + storage = @{ cc_backup_root = $script:ccRoot; nas_backup_path = $null } + network = @{ max_bandwidth_mbps = 0; robocopy_ipg_ms = 0 } + notifications = @{ enabled = $NotificationsEnabled; on_restore = $true } + } + } + + # Full set of "restore succeeds" mocks. Contexts re-Mock one seam to fail. $ChainManifestPath is + # the real on-disk manifest the verify/extract loop reads; $Mismatches lets a Context fail verify. + function Set-RestoreHappyMocks { + param([hashtable]$GlobalConfig, [string[]]$ChainManifestPaths, [string[]]$Mismatches = @()) + + Mock Write-SEBLog {} -ModuleName RestoreEngine + + Mock New-SEBLockFile -ModuleName RestoreEngine { + [PSCustomObject]@{ Acquired = $true; LockFilePath = 'X:\lock'; Reason = $null; StaleLockBroken = $false } + } + Mock Remove-SEBLockFile -ModuleName RestoreEngine { $true } + + Mock Get-SEBGlobalConfig -ModuleName RestoreEngine { $GlobalConfig }.GetNewClosure() + Mock Get-SEBNodeConfig -ModuleName RestoreEngine { @{ node = @{ hostname = 'node01' } } } + Mock New-SEBSession -ModuleName RestoreEngine { New-FakeSession -Name 'SEBackup-node01' -Target 'node01' } + Mock Remove-SEBSession -ModuleName RestoreEngine {} + Mock Get-SEBInstanceConfig -ModuleName RestoreEngine { + @{ world_path = 'D:\Torch\Instance\Saves\MyWorld'; share_name = 'SEBackup$' } + } + + # One archive in the chain by default (a single full). ChainManifests are the REAL files. + Mock Test-SEBRestoreChain -ModuleName RestoreEngine { + [PSCustomObject]@{ + Valid = $true + ChainLength = $ChainManifestPaths.Count + ChainManifests = $ChainManifestPaths + ChainArchives = @($ChainManifestPaths | ForEach-Object { ($_ -replace '\.json$', '.7z') }) + Errors = @() + Warnings = @() + } + }.GetNewClosure() + + # The safety backup succeeds and returns an archive path. + Mock Invoke-SEBBackup -ModuleName RestoreEngine { + [PSCustomObject]@{ Success = $true; ArchiveFile = 'C:\cc\PvPArena\full\safety.7z'; ErrorMessage = $null } + } + + # Real .OUTPUTS shapes: + # Stop-SEBTorchServer -> @{ Stopped; Method; ErrorMessage } + # Start-SEBTorchServer -> @{ Started; Method; APIResponding; ErrorMessage } + Mock Stop-SEBTorchServer -ModuleName RestoreEngine { @{ Stopped = $true; Method = 'service'; ErrorMessage = $null } } + Mock Start-SEBTorchServer -ModuleName RestoreEngine { @{ Started = $true; Method = 'service'; APIResponding = $true; ErrorMessage = $null } } + + Mock Deploy-SEBRestoredFiles -ModuleName RestoreEngine { + [PSCustomObject]@{ Deployed = $true; PreRestorePath = 'D:\Torch\Instance\Saves\MyWorld_prerestore_20260101_010101'; FilesCopied = 42; RolledBack = $null; ErrorMessage = $null } + } + + Mock Get-SEBSharePath -ModuleName RestoreEngine { '\\node01\SEBackup$' } + # Return Copy-SEBThrottled's REAL .OUTPUTS object (not nothing): the orchestrator pipes its + # result to | Out-Null, and only a realistic object proves that discard holds. If the Out-Null + # is dropped, this object leaks into Invoke-SEBRestore's output stream and @($r).Count -ne 1. + # Copy-SEBThrottled -> @{ Source; Destination; SizeBytes; DurationSeconds; AverageMbps; Method } + Mock Copy-SEBThrottled -ModuleName RestoreEngine { + [PSCustomObject]@{ Source = $Source; Destination = $Destination; SizeBytes = 12345; DurationSeconds = 1.0; AverageMbps = 98.7; Method = 'Robocopy' } + } + Mock Send-SEBRestoreNotification -ModuleName RestoreEngine {} + + # Reconstruction-verify remote block returns the mismatch set; the extract/cleanup/setup + # blocks are side-effect-only. Route by block text. + # + # FRAGILITY (this is the SAFETY-CRITICAL verify gate -- do NOT route it on one incidental + # literal silently): the verify block is the only remote call whose RESULT decides whether the + # restore proceeds to overwrite the live world. If its filter ever matched the WRONG block (or + # nothing), every restore test would route the verify call to the catch-all `$null` mock, a + # null .Mismatches would read as "no mismatches", and the deploy-abort-on-corrupt-reconstruction + # guarantee would go completely untested while the suite stayed green. To make the match robust + # we require TWO independent, defining markers of that block -- 'Get-FileHash' (the verification + # operation itself) AND 'Mismatches' (its return key) -- so a rename of either alone does not + # silently misroute it. The two markers are inlined into each filter (not factored into a shared + # variable) so the filter scriptblock closes over nothing and cannot break if that variable + # falls out of the mock-invocation scope. If you change the verify block in Invoke-SEBRestore.ps1, + # update BOTH markers here in lock-step; the catch-all is keyed as the negation of the same pair. + Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { + $ScriptBlock.ToString() -match 'Get-FileHash' -and $ScriptBlock.ToString() -match 'Mismatches' + } { + @{ Checked = 1; Mismatches = $Mismatches; Total = 1 } + }.GetNewClosure() + Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { + -not ($ScriptBlock.ToString() -match 'Get-FileHash' -and $ScriptBlock.ToString() -match 'Mismatches') + } { $null } + # The archive-local-path lookup and deleted-files/cleanup use raw Invoke-Command. + Mock Invoke-Command -ModuleName RestoreEngine { $null } + } +} + +AfterAll { + Remove-Item -LiteralPath $script:ccRoot -Recurse -Force -ErrorAction SilentlyContinue +} + +# =========================================================================================== +# HAPPY PATH +# =========================================================================================== +Describe 'Invoke-SEBRestore happy path' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_20260227_100000.json' -Type 'full' -ChainId 'rc-1' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) + } + + It 'returns Success with the result object populated (RestorePoint, UndoAvailable, Duration)' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_20260227_100000.json' -Force + + # Single-object .OUTPUTS contract: the side-effect remote blocks and Copy-SEBThrottled must + # not leak into the output stream and turn the caller's $result into an array. + @($r).Count | Should -Be 1 + $r | Should -BeOfType ([System.Management.Automation.PSCustomObject]) + $r.Success | Should -BeTrue + $r.RestorePoint | Should -Be 'PvPArena_FULL_20260227_100000.json' + $r.UndoAvailable | Should -BeTrue + $r.SafetyBackupPath | Should -Not -BeNullOrEmpty + $r.ErrorMessage | Should -BeNullOrEmpty + $r.Duration | Should -Not -BeNullOrEmpty + } + + It 'runs the steps in order: chain validate -> safety backup -> deploy -> start; stop is BEFORE deploy' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_20260227_100000.json' -Force | Out-Null + Should -Invoke Test-SEBRestoreChain -ModuleName RestoreEngine -Times 1 -Exactly + Should -Invoke Invoke-SEBBackup -ModuleName RestoreEngine -Times 1 -Exactly + Should -Invoke Stop-SEBTorchServer -ModuleName RestoreEngine -Times 1 -Exactly + Should -Invoke Deploy-SEBRestoredFiles -ModuleName RestoreEngine -Times 1 -Exactly + Should -Invoke Start-SEBTorchServer -ModuleName RestoreEngine -Times 1 -Exactly + } + + It 'takes the safety backup re-entrantly (-SkipLock -KeepSession) so it shares the restore lock/session' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_20260227_100000.json' -Force | Out-Null + # Issue #9 / re-entrancy: the nested safety backup must NOT take the lock again or close the + # session this restore keeps using for the destructive steps that follow. + Should -Invoke Invoke-SEBBackup -ModuleName RestoreEngine -Times 1 -Exactly -ParameterFilter { + $SkipLock -and $KeepSession -and $ForceFull + } + } + + It 'sends a restore notification via the restore notifier (not the backup one) with -InstanceName' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_20260227_100000.json' -Force | Out-Null + Should -Invoke Send-SEBRestoreNotification -ModuleName RestoreEngine -Times 1 -Exactly -ParameterFilter { + $InstanceName -eq 'PvPArena' + } + } + + It 'always releases the per-instance lock on the success path' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_20260227_100000.json' -Force | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly -ParameterFilter { $InstanceName -eq 'PvPArena' } + } +} + +# =========================================================================================== +# FAILURE INJECTION +# =========================================================================================== +Describe 'Invoke-SEBRestore failure injection' { + + Context 'lock already held -> aborts before any chain/safety/node work' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_lock.json' -Type 'full' -ChainId 'rc-lock' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) + Mock New-SEBLockFile -ModuleName RestoreEngine { + [PSCustomObject]@{ Acquired = $false; LockFilePath = 'X:\lock'; Reason = 'a backup is running'; StaleLockBroken = $false } + } + } + + It 'returns failure mentioning the lock and never validates the chain or runs the safety backup' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_lock.json' -Force + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'lock' + Should -Invoke Test-SEBRestoreChain -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Invoke-SEBBackup -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Stop-SEBTorchServer -ModuleName RestoreEngine -Times 0 -Exactly + } + + It 'does NOT release a lock it never acquired' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_lock.json' -Force | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 0 -Exactly + } + } + + Context 'chain invalid -> aborts before the safety backup and before any node work' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_badchain.json' -Type 'full' -ChainId 'rc-bad' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) + Mock Test-SEBRestoreChain -ModuleName RestoreEngine { + [PSCustomObject]@{ Valid = $false; ChainLength = 0; ChainManifests = @(); ChainArchives = @(); Errors = @('missing archive for sequence 1'); Warnings = @() } + } + } + + It 'returns failure with the chain error and runs no safety backup, stop, or deploy' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_badchain.json' -Force + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'chain validation failed' + $r.ErrorMessage | Should -Match 'missing archive' + Should -Invoke Invoke-SEBBackup -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Stop-SEBTorchServer -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Deploy-SEBRestoredFiles -ModuleName RestoreEngine -Times 0 -Exactly + } + + It 'still releases the lock (acquired before chain validation)' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_badchain.json' -Force | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly + } + } + + Context 'safety backup fails -> aborts BEFORE the live world is touched (no stop, no deploy)' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_safety.json' -Type 'full' -ChainId 'rc-safety' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) + Mock Invoke-SEBBackup -ModuleName RestoreEngine { + [PSCustomObject]@{ Success = $false; ArchiveFile = $null; ErrorMessage = 'node ran out of disk during safety backup' } + } + } + + It 'returns failure citing the safety backup and never stops the server or deploys' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_safety.json' -Force + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'Safety backup' + # The world must be untouched: no stop, no deploy, no start. + Should -Invoke Stop-SEBTorchServer -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Deploy-SEBRestoredFiles -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Start-SEBTorchServer -ModuleName RestoreEngine -Times 0 -Exactly + } + + It 'still releases the lock after the safety-backup abort' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_safety.json' -Force | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly + } + } + + Context 'reconstruction verification fails -> aborts BEFORE deploy (server never stopped)' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_verify.json' -Type 'full' -ChainId 'rc-verify' -Seq 0 + # The verify remote block reports a hash mismatch -> the orchestrator must abort pre-deploy. + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) -Mismatches @('HASH MISMATCH: Sandbox.sbc') + } + + It 'returns failure citing reconstruction verification and leaves the live world + server untouched' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_verify.json' -Force + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'verification failed' + # Critical: the server is stopped only AFTER a good reconstruction. A failed verify must + # leave the live world and server untouched -- NOTHING destructive may have run: no stop, + # no deploy, no start. (UndoAvailable must stay false: no prerestore dir was created.) + Should -Invoke Stop-SEBTorchServer -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Deploy-SEBRestoredFiles -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Start-SEBTorchServer -ModuleName RestoreEngine -Times 0 -Exactly + $r.UndoAvailable | Should -BeFalse + } + + It 'still releases the lock after the verify abort' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_verify.json' -Force | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly + } + } + + Context 'deploy fails -> structured failure surfaced; lock released' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_deploy.json' -Type 'full' -ChainId 'rc-deploy' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) + # Deploy-SEBRestoredFiles owns the prerestore-rename rollback internally; here it reports a + # deploy failure (already rolled back), which the orchestrator turns into a thrown abort. + Mock Deploy-SEBRestoredFiles -ModuleName RestoreEngine { + [PSCustomObject]@{ Deployed = $false; PreRestorePath = 'D:\...\MyWorld_prerestore_x'; FilesCopied = 0; RolledBack = $true; ErrorMessage = 'copy to world path failed; rolled back' } + } + } + + It 'returns failure with the deploy error message (Deployment failed: ...)' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_deploy.json' -Force + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'Deployment failed' + $r.ErrorMessage | Should -Match 'rolled back' + # The server WAS stopped (deploy runs after stop), but Start is not reached on a deploy throw. + Should -Invoke Stop-SEBTorchServer -ModuleName RestoreEngine -Times 1 -Exactly + Should -Invoke Start-SEBTorchServer -ModuleName RestoreEngine -Times 0 -Exactly + } + + It 'still releases the lock after a deploy failure (finally)' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_deploy.json' -Force | Out-Null + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly + } + } + + Context 'stop server fails (non-manual) -> aborts before deploy' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_stop.json' -Type 'full' -ChainId 'rc-stop' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) + Mock Stop-SEBTorchServer -ModuleName RestoreEngine { @{ Stopped = $false; Method = 'service'; ErrorMessage = 'service would not stop' } } + } + + It 'returns failure citing the stop and never deploys' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_stop.json' -Force + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'stop Torch server' + Should -Invoke Deploy-SEBRestoredFiles -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly + } + } +} + +# =========================================================================================== +# SAFETY-BACKUP OPT-OUT + START-SERVER DEGRADED + NOTIFICATION GATING +# =========================================================================================== +Describe 'Invoke-SEBRestore -SkipSafetyBackup and degraded-start handling' { + + Context '-SkipSafetyBackup skips the nested backup but still restores' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_skipsafety.json' -Type 'full' -ChainId 'rc-skip' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) + } + + It 'does not call Invoke-SEBBackup, warns about the skip, and still deploys + succeeds' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_skipsafety.json' -Force -SkipSafetyBackup + $r.Success | Should -BeTrue + ($r.Warnings -join ' ') | Should -Match 'Safety backup was skipped' + Should -Invoke Invoke-SEBBackup -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Deploy-SEBRestoredFiles -ModuleName RestoreEngine -Times 1 -Exactly + } + } + + Context 'server starts but API not responding -> success with a degraded-start warning' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_degraded.json' -Type 'full' -ChainId 'rc-deg' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) + Mock Start-SEBTorchServer -ModuleName RestoreEngine { @{ Started = $true; APIResponding = $false; ErrorMessage = $null } } + } + + It 'still reports Success but records the API-not-responding warning' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_degraded.json' -Force + $r.Success | Should -BeTrue + ($r.Warnings -join ' ') | Should -Match 'API is not responding' + } + } + + Context 'notifications disabled -> no restore notification sent' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_nonotify.json' -Type 'full' -ChainId 'rc-non' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig -NotificationsEnabled $false) -ChainManifestPaths @($man) + } + + It 'suppresses the restore notification when notifications.enabled=$false' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_nonotify.json' -Force + $r.Success | Should -BeTrue + Should -Invoke Send-SEBRestoreNotification -ModuleName RestoreEngine -Times 0 -Exactly + } + } +} + +# =========================================================================================== +# CONFIRMATION PROMPT (ShouldContinue substitute via Read-Host) -- -Force bypasses it +# =========================================================================================== +Describe 'Invoke-SEBRestore confirmation prompt' { + BeforeAll { + $man = New-ChainManifestFile -Dir $script:manifestsDir -Name 'PvPArena_FULL_confirm.json' -Type 'full' -ChainId 'rc-conf' -Seq 0 + Set-RestoreHappyMocks -GlobalConfig (New-RestoreGlobalConfig) -ChainManifestPaths @($man) + # Without -Force the orchestrator prompts via Read-Host. Mock it to decline. + Mock Read-Host -ModuleName RestoreEngine { 'no' } + } + + It 'without -Force, a declined prompt aborts before any chain/safety/node work' { + $r = Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_confirm.json' + $r.Success | Should -BeFalse + $r.ErrorMessage | Should -Match 'cancelled' + Should -Invoke Read-Host -ModuleName RestoreEngine -Times 1 -Exactly + Should -Invoke Invoke-SEBBackup -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Stop-SEBTorchServer -ModuleName RestoreEngine -Times 0 -Exactly + # Lock was acquired before the prompt, so it must still be released. + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly + } + + It 'does not consult the prompt at all when -Force is supplied' { + Invoke-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' -RestorePoint 'PvPArena_FULL_confirm.json' -Force | Out-Null + Should -Invoke Read-Host -ModuleName RestoreEngine -Times 0 -Exactly + } +} diff --git a/Tests/RestoreEngine/Undo-SEBRestore.Tests.ps1 b/Tests/RestoreEngine/Undo-SEBRestore.Tests.ps1 index bb5c3b1..ca2a16a 100644 --- a/Tests/RestoreEngine/Undo-SEBRestore.Tests.ps1 +++ b/Tests/RestoreEngine/Undo-SEBRestore.Tests.ps1 @@ -15,46 +15,98 @@ # * The downstream helpers (Get-SEBInstanceConfig, Stop/Start-SEBTorchServer, Invoke-SEBRemoteCommand) # strongly type -Session as [System.Management.Automation.Runspaces.PSSession]; PowerShell # enforces that cast during parameter binding even for mocked commands, and the type has no -# public constructor, so the New-SEBSession mock returns an *uninitialized* PSSession instance -# (which satisfies the cast). Its internals are never touched because every consumer is mocked. +# public constructor, so New-FakeSession (Tests/_TestHelpers/Test-Doubles.ps1) returns an +# *uninitialized* PSSession instance (which satisfies the cast). Its internals are never touched +# because every consumer is mocked. +# * The two Invoke-SEBRemoteCommand calls (prerestore-find, then rename-undo) are routed by their +# script-block SOURCE TEXT via ParameterFilter. That text is the disambiguation marker: the find +# block contains the 'prerestore_*' Get-ChildItem filter literal; the rename block contains the +# 'postrestore' suffix literal. These markers are load-bearing -- if the production source ever +# renames those literals, update the filters here in lock-step (a stale filter would silently +# route BOTH calls to one mock and quietly defeat the step-sequencing assertions). BeforeAll { $repoRoot = (Resolve-Path "$PSScriptRoot/../..").Path Import-Module "$repoRoot/SEBackup.psd1" -Force -DisableNameChecking 3>$null + . "$repoRoot/Tests/_TestHelpers/Test-Doubles.ps1" + + # Hoisted shared mock preamble for the locking/session-lifecycle contexts. Every context + # previously re-inlined the same ~12 mocks; this installs the full "undo succeeds" set so a + # context only re-Mocks the ONE seam it wants to vary. Parameters: + # -Notifications drives the best-effort restore-notification gate (default off). + # -SessionPreexisted $true => a session was already cached, so Undo borrows the caller's and + # must NOT tear it down; $false => Undo creates (and thus owns) it. + # -FindResult / -UndoResult override the two routed Invoke-SEBRemoteCommand return shapes. + function Set-UndoHappyMocks { + param( + [bool]$Notifications = $false, + [bool]$SessionPreexisted = $false, + [hashtable]$FindResult, + [hashtable]$UndoResult, + [hashtable]$StopResult, + [hashtable]$StartResult + ) + + if (-not $FindResult) { + $FindResult = @{ Found = $true; Path = 'C:\Torch\Instance\Saves\MyWorld_prerestore_20260101_010101'; Name = 'MyWorld_prerestore_20260101_010101'; Error = $null } + } + if (-not $UndoResult) { + $UndoResult = @{ Success = $true; PostRestorePath = 'C:\Torch\Instance\Saves\MyWorld_postrestore_20260101_010102'; Error = $null } + } + if (-not $StopResult) { + $StopResult = @{ Stopped = $true; Method = 'service'; ErrorMessage = $null } + } + if (-not $StartResult) { + $StartResult = @{ Started = $true; APIResponding = $true; ErrorMessage = $null } + } + + Mock Write-SEBLog {} -ModuleName RestoreEngine + + Mock New-SEBLockFile -ModuleName RestoreEngine { + [PSCustomObject]@{ Acquired = $true; LockFilePath = 'X:\lock'; Reason = $null; StaleLockBroken = $false } + } + Mock Remove-SEBLockFile -ModuleName RestoreEngine { $true } + + Mock Get-SEBGlobalConfig -ModuleName RestoreEngine { @{ notifications = @{ enabled = $Notifications; on_restore = $true } } }.GetNewClosure() + Mock Get-SEBNodeConfig -ModuleName RestoreEngine { @{ node = @{ hostname = 'node01' } } } + Mock Test-SEBSessionExists -ModuleName RestoreEngine { $SessionPreexisted }.GetNewClosure() + Mock New-SEBSession -ModuleName RestoreEngine { New-FakeSession -Name 'SEBackup-node01' -Target 'node01' } + Mock Remove-SEBSession -ModuleName RestoreEngine {} + Mock Get-SEBInstanceConfig -ModuleName RestoreEngine { @{ world_path = 'C:\Torch\Instance\Saves\MyWorld' } } + Mock Stop-SEBTorchServer -ModuleName RestoreEngine { $StopResult }.GetNewClosure() + Mock Start-SEBTorchServer -ModuleName RestoreEngine { $StartResult }.GetNewClosure() + Mock Send-SEBRestoreNotification -ModuleName RestoreEngine {} + + # Two Invoke-SEBRemoteCommand calls: the prerestore-find, then the rename undo. Distinguish + # them by the script-block SOURCE TEXT via ParameterFilter (see file header for the markers). + Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { $ScriptBlock.ToString() -match 'prerestore_\*' } { $FindResult }.GetNewClosure() + Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { $ScriptBlock.ToString() -match 'postrestore' } { $UndoResult }.GetNewClosure() + } + + # Builds a fresh on-disk sandbox (live world dir + a sibling _prerestore_ dir, each seeded with + # sentinel content) for the rename/rollback body tests below. Defined here in the file-level + # BeforeAll so it is visible inside the run-time It blocks (a function defined in a Describe body + # would only exist at discovery time). Sentinel content lets each assertion tell whether the world + # name ends up holding the LIVE world (rollback won) or the prerestore content. + $script:liveContent = 'LIVE-WORLD-CONTENT' + $script:preContent = 'PRERESTORE-CONTENT' + function New-UndoSandbox { + $base = Join-Path ([System.IO.Path]::GetTempPath()) ("sebundo_" + [guid]::NewGuid().ToString('n')) + New-Item -ItemType Directory -Path $base -Force | Out-Null + $worldDir = Join-Path $base 'MyWorld' + $preDir = Join-Path $base 'MyWorld_prerestore_20260101_010101' + New-Item -ItemType Directory -Path $worldDir, $preDir -Force | Out-Null + Set-Content -LiteralPath (Join-Path $worldDir 'live.txt') -Value $script:liveContent -NoNewline + Set-Content -LiteralPath (Join-Path $preDir 'pre.txt') -Value $script:preContent -NoNewline + [PSCustomObject]@{ Base = $base; WorldDir = $worldDir; PreDir = $preDir } + } } Describe 'Undo-SEBRestore locking and session lifecycle' { Context 'happy path (undo succeeds)' { BeforeAll { - Mock Write-SEBLog {} -ModuleName RestoreEngine - - Mock New-SEBLockFile -ModuleName RestoreEngine { - [PSCustomObject]@{ Acquired = $true; LockFilePath = 'X:\lock'; Reason = $null; StaleLockBroken = $false } - } - Mock Remove-SEBLockFile -ModuleName RestoreEngine { $true } - - Mock Get-SEBGlobalConfig -ModuleName RestoreEngine { @{ notifications = @{ enabled = $false } } } - Mock Get-SEBNodeConfig -ModuleName RestoreEngine { @{ node = @{ hostname = 'node01' } } } - # No session was cached before Undo runs, so Undo creates (and thus owns) it. - Mock Test-SEBSessionExists -ModuleName RestoreEngine { $false } - Mock New-SEBSession -ModuleName RestoreEngine { - [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject( - [System.Management.Automation.Runspaces.PSSession]) - } - Mock Remove-SEBSession -ModuleName RestoreEngine {} - Mock Get-SEBInstanceConfig -ModuleName RestoreEngine { @{ world_path = 'C:\Torch\Instance\Saves\MyWorld' } } - Mock Stop-SEBTorchServer -ModuleName RestoreEngine { @{ Stopped = $true; Method = 'service'; ErrorMessage = $null } } - Mock Start-SEBTorchServer -ModuleName RestoreEngine { @{ Started = $true; APIResponding = $true; ErrorMessage = $null } } - - # Two Invoke-SEBRemoteCommand calls: the prerestore-find, then the rename undo. Distinguish - # them by the script block text via ParameterFilter. - Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { $ScriptBlock.ToString() -match 'prerestore_\*' } { - @{ Found = $true; Path = 'C:\Torch\Instance\Saves\MyWorld_prerestore_20260101_010101'; Name = 'MyWorld_prerestore_20260101_010101'; Error = $null } - } - Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { $ScriptBlock.ToString() -match 'postrestore' } { - @{ Success = $true; PostRestorePath = 'C:\Torch\Instance\Saves\MyWorld_postrestore_20260101_010102'; Error = $null } - } + Set-UndoHappyMocks } It 'succeeds, acquires and releases the lock, and tears down its session' { @@ -69,33 +121,8 @@ Describe 'Undo-SEBRestore locking and session lifecycle' { Context 'undo fails partway (rename throws) but cleanup still runs' { BeforeAll { - Mock Write-SEBLog {} -ModuleName RestoreEngine - - Mock New-SEBLockFile -ModuleName RestoreEngine { - [PSCustomObject]@{ Acquired = $true; LockFilePath = 'X:\lock'; Reason = $null; StaleLockBroken = $false } - } - Mock Remove-SEBLockFile -ModuleName RestoreEngine { $true } - - Mock Get-SEBGlobalConfig -ModuleName RestoreEngine { @{ notifications = @{ enabled = $false } } } - Mock Get-SEBNodeConfig -ModuleName RestoreEngine { @{ node = @{ hostname = 'node01' } } } - # No session was cached before Undo runs, so Undo creates (and thus owns) it. - Mock Test-SEBSessionExists -ModuleName RestoreEngine { $false } - Mock New-SEBSession -ModuleName RestoreEngine { - [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject( - [System.Management.Automation.Runspaces.PSSession]) - } - Mock Remove-SEBSession -ModuleName RestoreEngine {} - Mock Get-SEBInstanceConfig -ModuleName RestoreEngine { @{ world_path = 'C:\Torch\Instance\Saves\MyWorld' } } - Mock Stop-SEBTorchServer -ModuleName RestoreEngine { @{ Stopped = $true; Method = 'service'; ErrorMessage = $null } } - Mock Start-SEBTorchServer -ModuleName RestoreEngine { @{ Started = $true; APIResponding = $true; ErrorMessage = $null } } - - Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { $ScriptBlock.ToString() -match 'prerestore_\*' } { - @{ Found = $true; Path = 'C:\Torch\Instance\Saves\MyWorld_prerestore_20260101_010101'; Name = 'MyWorld_prerestore_20260101_010101'; Error = $null } - } # The rename undo returns a failure result, which the function turns into a throw. - Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { $ScriptBlock.ToString() -match 'postrestore' } { - @{ Success = $false; PostRestorePath = $null; Error = 'Failed to rename prerestore directory back: simulated' } - } + Set-UndoHappyMocks -UndoResult @{ Success = $false; PostRestorePath = $null; Error = 'Failed to rename prerestore directory back: simulated' } } It 'reports failure but still releases the lock and tears down the session in finally' { @@ -110,26 +137,11 @@ Describe 'Undo-SEBRestore locking and session lifecycle' { Context 'lock cannot be acquired' { BeforeAll { - Mock Write-SEBLog {} -ModuleName RestoreEngine - + Set-UndoHappyMocks + # Re-Mock the lock to be unacquirable; everything downstream must be unreached. Mock New-SEBLockFile -ModuleName RestoreEngine { [PSCustomObject]@{ Acquired = $false; LockFilePath = 'X:\lock'; Reason = 'already locked'; StaleLockBroken = $false } } - Mock Remove-SEBLockFile -ModuleName RestoreEngine { $true } - - # None of these must be reached when the lock cannot be acquired. - Mock Get-SEBGlobalConfig -ModuleName RestoreEngine { @{ notifications = @{ enabled = $false } } } - Mock Get-SEBNodeConfig -ModuleName RestoreEngine { @{ node = @{ hostname = 'node01' } } } - Mock Test-SEBSessionExists -ModuleName RestoreEngine { $false } - Mock New-SEBSession -ModuleName RestoreEngine { - [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject( - [System.Management.Automation.Runspaces.PSSession]) - } - Mock Remove-SEBSession -ModuleName RestoreEngine {} - Mock Get-SEBInstanceConfig -ModuleName RestoreEngine { @{ world_path = 'C:\Torch\Instance\Saves\MyWorld' } } - Mock Stop-SEBTorchServer -ModuleName RestoreEngine { @{ Stopped = $true; Method = 'service'; ErrorMessage = $null } } - Mock Start-SEBTorchServer -ModuleName RestoreEngine { @{ Started = $true; APIResponding = $true; ErrorMessage = $null } } - Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine { @{} } } It 'aborts before touching the world and never creates a session or releases a lock it never held' { @@ -147,35 +159,48 @@ Describe 'Undo-SEBRestore locking and session lifecycle' { } } - Context 'caller already owned a session (preexisting cached session is preserved)' { + Context 'no prerestore directory found -> structured failure before any rename' { BeforeAll { - Mock Write-SEBLog {} -ModuleName RestoreEngine + # The find returns Found=$false; the undo must NOT proceed to stop the server or rename. + Set-UndoHappyMocks -FindResult @{ Found = $false; Path = $null; Error = "No prerestore directories found for 'MyWorld'." } ` + -UndoResult @{ Success = $true; PostRestorePath = 'unexpected'; Error = $null } + } - Mock New-SEBLockFile -ModuleName RestoreEngine { - [PSCustomObject]@{ Acquired = $true; LockFilePath = 'X:\lock'; Reason = $null; StaleLockBroken = $false } - } - Mock Remove-SEBLockFile -ModuleName RestoreEngine { $true } - - Mock Get-SEBGlobalConfig -ModuleName RestoreEngine { @{ notifications = @{ enabled = $false } } } - Mock Get-SEBNodeConfig -ModuleName RestoreEngine { @{ node = @{ hostname = 'node01' } } } - # A session for this node was ALREADY cached before Undo ran. New-SEBSession - # therefore hands back the caller's existing session; Undo does NOT own it. - Mock Test-SEBSessionExists -ModuleName RestoreEngine { $true } - Mock New-SEBSession -ModuleName RestoreEngine { - [System.Runtime.Serialization.FormatterServices]::GetUninitializedObject( - [System.Management.Automation.Runspaces.PSSession]) - } - Mock Remove-SEBSession -ModuleName RestoreEngine {} - Mock Get-SEBInstanceConfig -ModuleName RestoreEngine { @{ world_path = 'C:\Torch\Instance\Saves\MyWorld' } } - Mock Stop-SEBTorchServer -ModuleName RestoreEngine { @{ Stopped = $true; Method = 'service'; ErrorMessage = $null } } - Mock Start-SEBTorchServer -ModuleName RestoreEngine { @{ Started = $true; APIResponding = $true; ErrorMessage = $null } } + It 'fails with the no-prerestore error and never stops the server or runs the rename' { + $result = Undo-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' + $result.Success | Should -BeFalse + $result.ErrorMessage | Should -Match 'No prerestore' + # The find ran, but the server stop and the rename undo must not. + Should -Invoke Stop-SEBTorchServer -ModuleName RestoreEngine -Times 0 -Exactly + Should -Invoke Invoke-SEBRemoteCommand -ModuleName RestoreEngine -Times 0 -Exactly -ParameterFilter { $ScriptBlock.ToString() -match 'postrestore' } + # Lock + session still cleaned up. + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName RestoreEngine -Times 1 -Exactly + } + } - Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { $ScriptBlock.ToString() -match 'prerestore_\*' } { - @{ Found = $true; Path = 'C:\Torch\Instance\Saves\MyWorld_prerestore_20260101_010101'; Name = 'MyWorld_prerestore_20260101_010101'; Error = $null } - } - Mock Invoke-SEBRemoteCommand -ModuleName RestoreEngine -ParameterFilter { $ScriptBlock.ToString() -match 'postrestore' } { - @{ Success = $true; PostRestorePath = 'C:\Torch\Instance\Saves\MyWorld_postrestore_20260101_010102'; Error = $null } - } + Context 'stop server fails (non-manual) -> aborts before the rename, structured failure' { + BeforeAll { + # Stop fails with a non-manual method -> must abort before the world rename. + Set-UndoHappyMocks -StopResult @{ Stopped = $false; Method = 'service'; ErrorMessage = 'service would not stop' } ` + -UndoResult @{ Success = $true; PostRestorePath = 'should-not-happen'; Error = $null } + } + + It 'fails citing the stop and never runs the rename undo' { + $result = Undo-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' + $result.Success | Should -BeFalse + $result.ErrorMessage | Should -Match 'stop Torch server' + Should -Invoke Invoke-SEBRemoteCommand -ModuleName RestoreEngine -Times 0 -Exactly -ParameterFilter { $ScriptBlock.ToString() -match 'postrestore' } + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName RestoreEngine -Times 1 -Exactly + } + } + + Context 'caller already owned a session (preexisting cached session is preserved)' { + BeforeAll { + # A session for this node was ALREADY cached before Undo ran. New-SEBSession therefore + # hands back the caller's existing session; Undo does NOT own it and must not remove it. + Set-UndoHappyMocks -SessionPreexisted $true } It 'succeeds and releases the lock but does NOT tear down the caller-owned session' { @@ -188,4 +213,183 @@ Describe 'Undo-SEBRestore locking and session lifecycle' { Should -Invoke Remove-SEBSession -ModuleName RestoreEngine -Times 0 -Exactly } } + + Context 'happy path with notifications enabled sends an Undo restore notification' { + BeforeAll { + # notifications.enabled = $true so the best-effort restore notification fires. + Set-UndoHappyMocks -Notifications $true + } + + It 'succeeds and sends the Undo restore notification with InitiatedBy=Undo-SEBRestore' { + $result = Undo-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' + $result.Success | Should -BeTrue + $result.PostRestorePath | Should -Not -BeNullOrEmpty + Should -Invoke Send-SEBRestoreNotification -ModuleName RestoreEngine -Times 1 -Exactly -ParameterFilter { + $InstanceName -eq 'PvPArena' -and $InitiatedBy -eq 'Undo-SEBRestore' -and $RestorePoint -match 'Undo' + } + } + } + + Context 'start server fails after a successful rename -> still reports Success (undo already done)' { + BeforeAll { + # The rename already completed; a failed START must NOT flip the undo to a failure (the + # world was already swapped back -- reporting failure would be misleading). + Set-UndoHappyMocks -StartResult @{ Started = $false; APIResponding = $false; ErrorMessage = 'torch.exe did not launch' } + } + + It 'reports Success despite the start failure and still releases the lock + session' { + $result = Undo-SEBRestore -NodeName 'node01' -InstanceName 'PvPArena' + # Decisive: the rename undo succeeded, so the overall undo is a success even though the + # server failed to come back up (the operator restarts it manually). + $result.Success | Should -BeTrue + $result.PostRestorePath | Should -Not -BeNullOrEmpty + Should -Invoke Remove-SEBLockFile -ModuleName RestoreEngine -Times 1 -Exactly + Should -Invoke Remove-SEBSession -ModuleName RestoreEngine -Times 1 -Exactly + } + } +} + +# =========================================================================================== +# THE ROLLBACK SCRIPT BLOCK -- the single most dangerous path in the system, run for real. +# =========================================================================================== +# The undo's rename body (Undo-SEBRestore.ps1: move the live world aside to a _postrestore_ name, +# rename the _prerestore_ dir back to the world name, and ON A RENAME-BACK FAILURE roll the current +# world back into place) is node-local and runs inside a script block handed to Invoke-SEBRemoteCommand. +# The locking/lifecycle contexts above mock Invoke-SEBRemoteCommand to RETURN a canned result, so that +# body never executes there: deleting the rollback Rename-Item would not fail any of them. +# +# Pin the REAL body the way New-SEBManifest.Tests.ps1 pins its remote scan block: lift the literal +# script block out of the source via the AST and invoke it against REAL temp directories, with the +# rename-back FORCED to fail (an exclusive handle on a file inside the _prerestore_ dir makes Windows +# refuse to rename that directory). The decisive assertion is that the rollback puts the ORIGINAL live +# world content back under the world name -- NOT stranded under a _postrestore_ name. This runs the +# production code, not a copy: remove the rollback Rename-Item in the source and these go red (the +# live content ends up orphaned under _postrestore_ and the world name is left holding the prerestore +# dir's content, or nothing). +Describe 'Undo-SEBRestore rename/rollback body (real script block over temp dirs)' { + + BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../..").Path + $src = Join-Path $repoRoot 'Modules/RestoreEngine/Public/Undo-SEBRestore.ps1' + + # Lift the rename/rollback script block (the SECOND Invoke-SEBRemoteCommand call) straight out + # of the source. Disambiguate from the prerestore-find block by the rename-specific markers + # ('postRestoreName' + 'Rename-Item'); both are stable literals in the rollback body. + $tokens = $null; $parseErrors = $null + $fileAst = [System.Management.Automation.Language.Parser]::ParseFile($src, [ref]$tokens, [ref]$parseErrors) + $invokeCalls = $fileAst.FindAll( + { param($n) $n -is [System.Management.Automation.Language.CommandAst] -and $n.GetCommandName() -eq 'Invoke-SEBRemoteCommand' }, + $true) + $script:renameBlock = $null + foreach ($call in $invokeCalls) { + $sbExpr = $call.CommandElements | + Where-Object { $_ -is [System.Management.Automation.Language.ScriptBlockExpressionAst] } | + Select-Object -First 1 + if ($sbExpr -and $sbExpr.Extent.Text -match 'postRestoreName' -and $sbExpr.Extent.Text -match 'Rename-Item') { + # GetScriptBlock() yields a real, invokable [scriptblock] from the AST node. + $script:renameBlock = $sbExpr.ScriptBlock.GetScriptBlock() + break + } + } + if ($null -eq $script:renameBlock) { + throw "Could not lift the rename/rollback script block from Undo-SEBRestore.ps1 (markers 'postRestoreName' + 'Rename-Item')." + } + } + + Context 'rename-back FAILS -> the live world is rolled back into place (not stranded)' { + It 'returns a structured failure citing the rename-back' { + $sb = New-UndoSandbox + # Force the prerestore -> worldName rename to fail: hold an exclusive handle on a file + # inside the prerestore dir so Windows refuses to rename that directory. + $lockFile = Join-Path $sb.PreDir 'locked.bin' + Set-Content -LiteralPath $lockFile -Value 'x' -NoNewline + $fs = [System.IO.File]::Open($lockFile, 'Open', 'Read', 'None') + try { + $r = & $script:renameBlock -worldDir $sb.WorldDir -preRestorePath $sb.PreDir + } + finally { $fs.Close(); $fs.Dispose() } + + $r.Success | Should -BeFalse + $r.Error | Should -Match 'rename prerestore directory back' + + Remove-Item -LiteralPath $sb.Base -Recurse -Force -ErrorAction SilentlyContinue + } + + It 'restores the ORIGINAL live world content under the world name (rollback ran)' { + $sb = New-UndoSandbox + $lockFile = Join-Path $sb.PreDir 'locked.bin' + Set-Content -LiteralPath $lockFile -Value 'x' -NoNewline + $fs = [System.IO.File]::Open($lockFile, 'Open', 'Read', 'None') + try { + & $script:renameBlock -worldDir $sb.WorldDir -preRestorePath $sb.PreDir | Out-Null + } + finally { $fs.Close(); $fs.Dispose() } + + $liveFile = Join-Path $sb.WorldDir 'live.txt' + # DECISIVE (mutation target): the rollback Rename-Item must move the live world back under + # the world name. Remove that line in the source and this fails -- the live content is + # left orphaned under _postrestore_ and the world name no longer holds it. + Test-Path -LiteralPath $sb.WorldDir | Should -BeTrue -Because 'the world directory must exist again after rollback' + Test-Path -LiteralPath $liveFile | Should -BeTrue -Because 'the original live world content must be rolled back under the world name' + (Get-Content -LiteralPath $liveFile -Raw) | Should -Be $script:liveContent + + Remove-Item -LiteralPath $sb.Base -Recurse -Force -ErrorAction SilentlyContinue + } + + It 'does NOT leave the original world stranded under a _postrestore_ name' { + $sb = New-UndoSandbox + $lockFile = Join-Path $sb.PreDir 'locked.bin' + Set-Content -LiteralPath $lockFile -Value 'x' -NoNewline + $fs = [System.IO.File]::Open($lockFile, 'Open', 'Read', 'None') + try { + & $script:renameBlock -worldDir $sb.WorldDir -preRestorePath $sb.PreDir | Out-Null + } + finally { $fs.Close(); $fs.Dispose() } + + # After a successful rollback the moved-aside copy is gone (renamed back), so no + # _postrestore_ directory remains holding the live world. + $stranded = @(Get-ChildItem -LiteralPath $sb.Base -Directory -Filter 'MyWorld_postrestore_*' -ErrorAction SilentlyContinue) + $stranded.Count | Should -Be 0 -Because 'the rollback renamed the postrestore copy back to the world name' + + Remove-Item -LiteralPath $sb.Base -Recurse -Force -ErrorAction SilentlyContinue + } + } + + Context 'happy rename (both renames succeed) -> prerestore content lands under the world name' { + It 'reports Success and the world now holds the prerestore content with a postrestore copy of the old world' { + $sb = New-UndoSandbox + $r = & $script:renameBlock -worldDir $sb.WorldDir -preRestorePath $sb.PreDir + + $r.Success | Should -BeTrue + $r.PostRestorePath | Should -Not -BeNullOrEmpty + + # The world name now holds what WAS the prerestore content (pre.txt), and the old live + # world was moved aside under the returned _postrestore_ path. + $preFileNowInWorld = Join-Path $sb.WorldDir 'pre.txt' + Test-Path -LiteralPath $preFileNowInWorld | Should -BeTrue + (Get-Content -LiteralPath $preFileNowInWorld -Raw) | Should -Be $script:preContent + + Test-Path -LiteralPath $r.PostRestorePath | Should -BeTrue + (Get-Content -LiteralPath (Join-Path $r.PostRestorePath 'live.txt') -Raw) | Should -Be $script:liveContent + + Remove-Item -LiteralPath $sb.Base -Recurse -Force -ErrorAction SilentlyContinue + } + } + + Context 'no current world dir present -> renames prerestore back with no postrestore copy' { + It 'reports Success with a null PostRestorePath (nothing to move aside)' { + $sb = New-UndoSandbox + # Remove the live world entirely: the block must skip the move-aside and just rename the + # prerestore back, returning a null PostRestorePath. + Remove-Item -LiteralPath $sb.WorldDir -Recurse -Force + $r = & $script:renameBlock -worldDir $sb.WorldDir -preRestorePath $sb.PreDir + + $r.Success | Should -BeTrue + $r.PostRestorePath | Should -BeNullOrEmpty + # The prerestore content is now under the world name. + Test-Path -LiteralPath (Join-Path $sb.WorldDir 'pre.txt') | Should -BeTrue + + Remove-Item -LiteralPath $sb.Base -Recurse -Force -ErrorAction SilentlyContinue + } + } }