From d4cc5c08462016d775dc2b2ba87f8c11cce58b4b Mon Sep 17 00:00:00 2001 From: jessikitty Date: Fri, 12 Dec 2025 10:09:17 +1100 Subject: [PATCH] Add weekly live backup script with checkpoint-based zero-downtime backup --- weekly_vmbackup.ps1 | 646 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 646 insertions(+) create mode 100644 weekly_vmbackup.ps1 diff --git a/weekly_vmbackup.ps1 b/weekly_vmbackup.ps1 new file mode 100644 index 0000000..806ed5f --- /dev/null +++ b/weekly_vmbackup.ps1 @@ -0,0 +1,646 @@ +# Weekly VM Live Backup Script - Checkpoint-based with 7-Zip Compression +# Run Weekly on Sunday nights at 2am +# Location: Hyper-V Server +# Uses VM checkpoints for live backup without downtime + +param( + [Parameter(ParameterSetName='SingleVM', Mandatory=$true)] + [string]$VMName, + + [Parameter(ParameterSetName='AllVMs', Mandatory=$true)] + [switch]$AllVMs, + + [Parameter(ParameterSetName='AllVMs')] + [string[]]$ExcludeVMs = @(), + + [int]$MaxBackups = 4 +) + +# Configuration - Modify these paths as needed +$NASBackupPath = "\\nas5362\Backups\Weekly" +$TempBackupPath = "F:\TempBackup\Weekly" +$LocalBackupPath = "F:\Backups\Weekly" +$SevenZipPath = "C:\Program Files\7-Zip\7z.exe" + +# Configure logging +$LogPath = "C:\Scripts\Logs\Weekly_Backup_$(Get-Date -Format 'yyyyMM').log" +$null = New-Item -ItemType Directory -Path (Split-Path $LogPath -Parent) -Force -ErrorAction SilentlyContinue + +function Write-Log { + param([string]$Message, [string]$Level = "INFO") + $TimeStamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + $LogEntry = "[$TimeStamp] [$Level] $Message" + Write-Host $LogEntry + Add-Content -Path $LogPath -Value $LogEntry +} + +function Test-NetworkPath { + param([string]$Path) + try { + if (-not (Test-Path $Path -ErrorAction Stop)) { + Write-Log "Network path does not exist: $Path" "ERROR" + return $false + } + + $TestFile = Join-Path $Path "test_write_$(Get-Date -Format 'yyyyMMddHHmmss').tmp" + try { + $null = New-Item -ItemType File -Path $TestFile -Force -ErrorAction Stop + Remove-Item -Path $TestFile -Force -ErrorAction SilentlyContinue + Write-Log "Network path is accessible and writable: $Path" + return $true + } + catch { + Write-Log "Network path exists but is not writable: $Path - $_" "ERROR" + return $false + } + } + catch { + Write-Log "Network path test failed: $Path - $_" "ERROR" + return $false + } +} + +function Test-Prerequisites { + # Test 7-Zip installation + if (-not (Test-Path $SevenZipPath)) { + Write-Log "7-Zip not found at $SevenZipPath" "ERROR" + Write-Log "Please install 7-Zip or update the SevenZipPath variable" "ERROR" + return $false + } + + # Test temp backup directory + try { + $null = New-Item -ItemType Directory -Path $TempBackupPath -Force -ErrorAction Stop + Write-Log "Temp backup directory ready: $TempBackupPath" + } + catch { + Write-Log "Cannot create temp backup directory: $TempBackupPath - $_" "ERROR" + return $false + } + + # Test local backup directory + try { + $null = New-Item -ItemType Directory -Path $LocalBackupPath -Force -ErrorAction Stop + Write-Log "Local backup directory ready: $LocalBackupPath" + } + catch { + Write-Log "Cannot create local backup directory: $LocalBackupPath - $_" "ERROR" + return $false + } + + # Test NAS connectivity (but don't fail if it's temporarily unavailable) + if (-not (Test-NetworkPath $NASBackupPath)) { + Write-Log "NAS connectivity test failed - will attempt copy later" "WARNING" + } + + return $true +} + +function Compress-BackupWithSevenZip { + param( + [string]$SourcePath, + [string]$ArchivePath, + [string]$VMName + ) + + try { + Write-Log "Starting 7-Zip compression for $VMName..." + Write-Log "Source: $SourcePath" + Write-Log "Archive: $ArchivePath" + + # Ensure destination directory exists + $ArchiveDir = Split-Path $ArchivePath -Parent + $null = New-Item -ItemType Directory -Path $ArchiveDir -Force -ErrorAction SilentlyContinue + + # 7-Zip command: a = add, -t7z = 7z format, -mx=7 = high compression (faster than max for weekly), -mmt = multithreaded + $Arguments = @( + 'a', + '-t7z', + '-mx=7', + '-mmt', + "`"$ArchivePath`"", + "`"$SourcePath\*`"" + ) + + Write-Log "7-Zip command: `"$SevenZipPath`" $($Arguments -join ' ')" + + $StartTime = Get-Date + $Process = Start-Process -FilePath $SevenZipPath -ArgumentList $Arguments -Wait -PassThru -NoNewWindow + $EndTime = Get-Date + + $CompressionTime = ($EndTime - $StartTime).TotalMinutes + Write-Log "7-Zip compression completed in $([math]::Round($CompressionTime, 1)) minutes" + + if ($Process.ExitCode -eq 0) { + # Get compression statistics + if (Test-Path $ArchivePath) { + $OriginalSize = (Get-ChildItem -Path $SourcePath -Recurse -File | Measure-Object -Property Length -Sum).Sum + $CompressedSize = (Get-Item $ArchivePath).Length + $CompressionRatio = [math]::Round((1 - ($CompressedSize / $OriginalSize)) * 100, 1) + + $OriginalSizeGB = [math]::Round($OriginalSize / 1GB, 2) + $CompressedSizeGB = [math]::Round($CompressedSize / 1GB, 2) + + Write-Log "Compression successful for $VMName" + Write-Log "Original size: $OriginalSizeGB GB" + Write-Log "Compressed size: $CompressedSizeGB GB" + Write-Log "Compression ratio: $CompressionRatio%" + return $true + } + else { + Write-Log "7-Zip reported success but archive file not found" "ERROR" + return $false + } + } + else { + Write-Log "7-Zip compression failed with exit code: $($Process.ExitCode)" "ERROR" + return $false + } + } + catch { + Write-Log "Error during 7-Zip compression: $_" "ERROR" + return $false + } +} + +function Move-ToLocalBackup { + param( + [string]$SourceFile, + [string]$DestinationPath, + [string]$VMName + ) + + try { + Write-Log "Moving compressed backup to local backup folder for $VMName..." + + # Ensure destination directory exists + $null = New-Item -ItemType Directory -Path $DestinationPath -Force -ErrorAction SilentlyContinue + + $DestinationFile = Join-Path $DestinationPath (Split-Path $SourceFile -Leaf) + + # Move the file (much faster than copy since it's on the same drive) + Move-Item -Path $SourceFile -Destination $DestinationFile -Force + Write-Log "Successfully moved backup to local folder for $VMName" + + return $DestinationFile + } + catch { + Write-Log "Error moving to local backup folder: $_" "ERROR" + return $null + } +} + +function Copy-AllBackupsToNAS { + param( + [string]$LocalPath, + [string]$NASPath, + [array]$BackupFiles + ) + + try { + Write-Log "=== Starting bulk copy of all backups to NAS ===" + Write-Log "Copying $($BackupFiles.Count) backup file(s) to NAS..." + + # Test NAS connectivity before attempting bulk copy + if (-not (Test-NetworkPath $NASPath)) { + throw "NAS is not accessible for bulk copy" + } + + $SuccessfulCopies = 0 + $FailedCopies = 0 + $TotalSize = 0 + + foreach ($BackupFile in $BackupFiles) { + try { + Write-Log "Copying: $(Split-Path $BackupFile -Leaf)" + + # Use robocopy for reliable network transfer + $RobocopyArgs = @( + "`"$(Split-Path $BackupFile -Parent)`"", + "`"$NASPath`"", + "`"$(Split-Path $BackupFile -Leaf)`"", + "/Z", # Copy files in restartable mode + "/R:3", # Retry 3 times on failed copies + "/W:30", # Wait 30 seconds between retries + "/TBD", # Wait for share names to be defined + "/NP" # No progress display + ) + + $StartTime = Get-Date + $Process = Start-Process -FilePath "robocopy.exe" -ArgumentList $RobocopyArgs -Wait -PassThru -NoNewWindow + $EndTime = Get-Date + + $CopyTime = ($EndTime - $StartTime).TotalSeconds + + # Robocopy exit codes: 0-3 are success + if ($Process.ExitCode -le 3) { + $FileSize = (Get-Item $BackupFile).Length + $TotalSize += $FileSize + $FileSizeMB = [math]::Round($FileSize / 1MB, 1) + Write-Log "Successfully copied $(Split-Path $BackupFile -Leaf) ($FileSizeMB MB) in $CopyTime seconds" + $SuccessfulCopies++ + } + else { + Write-Log "Failed to copy $(Split-Path $BackupFile -Leaf). Robocopy exit code: $($Process.ExitCode)" "ERROR" + $FailedCopies++ + } + } + catch { + Write-Log "Error copying $(Split-Path $BackupFile -Leaf): $_" "ERROR" + $FailedCopies++ + } + } + + $TotalSizeGB = [math]::Round($TotalSize / 1GB, 2) + Write-Log "=== Bulk copy completed ===" + Write-Log "Successful copies: $SuccessfulCopies" + Write-Log "Failed copies: $FailedCopies" + Write-Log "Total data copied: $TotalSizeGB GB" + + return $FailedCopies -eq 0 + } + catch { + Write-Log "Error during bulk copy to NAS: $_" "ERROR" + return $false + } +} + +function Cleanup-TempFiles { + param( + [string]$TempVMPath, + [string]$TempArchive, + [string]$VMName + ) + + try { + Write-Log "Cleaning up temporary files for $VMName..." + + # Remove temp VM export folder + if ($TempVMPath -and (Test-Path $TempVMPath)) { + Remove-Item -Path $TempVMPath -Recurse -Force + Write-Log "Removed temp VM folder: $TempVMPath" + } + + # Remove temp archive + if ($TempArchive -and (Test-Path $TempArchive)) { + Remove-Item -Path $TempArchive -Force + Write-Log "Removed temp archive: $TempArchive" + } + } + catch { + Write-Log "Warning: Could not cleanup some temp files for $VMName : $_" "WARNING" + } +} + +function Cleanup-OldBackups { + param([string]$Path, [string]$Pattern, [int]$Keep) + + try { + # Look for 7z archives + $BackupFiles = Get-ChildItem -Path $Path -File -Filter "*.7z" | + Where-Object { $_.Name -match $Pattern } | + Sort-Object CreationTime -Descending + + if ($BackupFiles.Count -gt $Keep) { + $ToDelete = $BackupFiles | Select-Object -Skip $Keep + foreach ($File in $ToDelete) { + Write-Log "Deleting old backup: $($File.FullName)" + Remove-Item -Path $File.FullName -Force + } + } + $KeptCount = if ($BackupFiles.Count -lt $Keep) { $BackupFiles.Count } else { $Keep } + Write-Log "Cleanup completed. Kept $KeptCount backups" + } + catch { + Write-Log "Error during cleanup: $_" "ERROR" + throw + } +} + +function Backup-SingleVM { + param( + [string]$VMName, + [int]$MaxBackups + ) + + $VMBackupSuccess = $false + $CheckpointName = $null + $TempVMFolder = $null + $TempArchive = $null + $LocalBackupFile = $null + + try { + Write-Log "=== Starting LIVE backup for VM: $VMName ===" + + # Verify VM exists + $VM = Get-VM -Name $VMName -ErrorAction Stop + Write-Log "VM found: $($VM.Name) - State: $($VM.State)" + + # Create temp folders for this VM + $VMTempFolder = Join-Path $TempBackupPath $VMName + $TempVMFolder = Join-Path $VMTempFolder "Export" + $TempArchive = Join-Path $VMTempFolder "Weekly_$($VMName)_$(Get-Date -Format 'yyyyMMdd_HHmm').7z" + + # Ensure temp directories exist + $null = New-Item -ItemType Directory -Path $TempVMFolder -Force + Write-Log "Created temp export folder: $TempVMFolder" + + # Create a checkpoint for consistent backup point + $CheckpointName = "WeeklyBackup_$(Get-Date -Format 'yyyyMMdd_HHmmss')" + Write-Log "Creating checkpoint: $CheckpointName" + + try { + Checkpoint-VM -Name $VMName -SnapshotName $CheckpointName + Write-Log "Checkpoint created successfully" + } + catch { + # If checkpoint creation fails, we'll continue with live export (less consistent but still functional) + Write-Log "Checkpoint creation failed, proceeding with live export: $_" "WARNING" + $CheckpointName = $null + } + + # Export VM to temp folder (this works on running VMs) + Write-Log "Starting live VM export to temp folder..." + + try { + # Use Export-VM with the running VM - Hyper-V handles this gracefully + $ExportJob = Export-VM -Name $VMName -Path $TempVMFolder -AsJob + + # Monitor export progress + do { + Start-Sleep -Seconds 30 + $Progress = Get-Job -Id $ExportJob.Id + Write-Log "Live export status: $($Progress.State)" + + # Check if VM is still accessible during export + try { + $VMCheck = Get-VM -Name $VMName -ErrorAction SilentlyContinue + if ($VMCheck) { + Write-Log "VM state during export: $($VMCheck.State)" + } + } + catch { + Write-Log "Could not check VM state during export" "WARNING" + } + + } while ($Progress.State -eq 'Running') + + # Check export result + $ExportResult = Receive-Job -Job $ExportJob + Remove-Job -Job $ExportJob + + if ($Progress.State -eq 'Completed') { + # Verify the export actually succeeded by checking for VM files + $VMExportFolder = Get-ChildItem -Path $TempVMFolder -Directory -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($VMExportFolder) { + $VHDXFiles = Get-ChildItem -Path $VMExportFolder.FullName -Recurse -Filter "*.vhdx" -ErrorAction SilentlyContinue + $VMCXFiles = Get-ChildItem -Path $VMExportFolder.FullName -Recurse -Filter "*.vmcx" -ErrorAction SilentlyContinue + $AVHDXFiles = Get-ChildItem -Path $VMExportFolder.FullName -Recurse -Filter "*.avhdx" -ErrorAction SilentlyContinue + + if ($VHDXFiles.Count -eq 0 -and $VMCXFiles.Count -eq 0 -and $AVHDXFiles.Count -eq 0) { + throw "Export job completed but no VM files were found in temp folder" + } + + Write-Log "Live VM export completed successfully" + Write-Log "Found $($VHDXFiles.Count) VHDX file(s), $($VMCXFiles.Count) VMCX file(s), and $($AVHDXFiles.Count) AVHDX file(s)" + } else { + throw "Export job completed but no VM folder was created in temp directory" + } + } + else { + throw "Live VM export failed with job state: $($Progress.State)" + } + } + catch { + Write-Log "Live export failed: $_" "ERROR" + throw + } + + # Remove the checkpoint now that export is complete + if ($CheckpointName) { + try { + Write-Log "Removing backup checkpoint: $CheckpointName" + Remove-VMCheckpoint -VMName $VMName -Name $CheckpointName -IncludeAllChildCheckpoints + Write-Log "Checkpoint removed successfully" + } + catch { + Write-Log "Warning: Could not remove checkpoint $CheckpointName : $_" "WARNING" + } + } + + # Verify VM is still running properly + try { + $FinalVMCheck = Get-VM -Name $VMName + Write-Log "VM final status after backup: $($FinalVMCheck.State)" + } + catch { + Write-Log "Warning: Could not verify final VM state" "WARNING" + } + + # Compress the backup with 7-Zip (using slightly less compression for speed) + if (-not (Compress-BackupWithSevenZip -SourcePath $TempVMFolder -ArchivePath $TempArchive -VMName $VMName)) { + throw "7-Zip compression failed" + } + + # Move compressed backup to local backup folder + $LocalBackupFile = Move-ToLocalBackup -SourceFile $TempArchive -DestinationPath $LocalBackupPath -VMName $VMName + if (-not $LocalBackupFile) { + throw "Failed to move backup to local backup folder" + } + + # Immediately cleanup temp files for this VM after successful compression and move + Write-Log "Cleaning up temp files for $VMName after successful compression..." + Cleanup-TempFiles -TempVMPath $TempVMFolder -TempArchive $TempArchive -VMName $VMName + + $VMBackupSuccess = $true + Write-Log "=== Weekly LIVE backup completed successfully for $VMName ===" "SUCCESS" + + } + catch { + Write-Log "=== LIVE backup failed for $VMName : $_ ===" "ERROR" + + # Attempt to remove checkpoint if it was created + if ($CheckpointName) { + try { + Write-Log "Attempting to remove checkpoint after failure..." + Remove-VMCheckpoint -VMName $VMName -Name $CheckpointName -IncludeAllChildCheckpoints -ErrorAction SilentlyContinue + Write-Log "Checkpoint cleanup completed" + } + catch { + Write-Log "Could not remove checkpoint after failure: $_" "WARNING" + } + } + } + finally { + # Always cleanup temp files for this VM (backup in case not already done) + if ((Test-Path $TempVMFolder -ErrorAction SilentlyContinue) -or (Test-Path $TempArchive -ErrorAction SilentlyContinue)) { + Write-Log "Final cleanup of any remaining temp files for $VMName..." + Cleanup-TempFiles -TempVMPath $TempVMFolder -TempArchive $TempArchive -VMName $VMName + } + } + + # Return both success status and local backup file path + return @{ + Success = $VMBackupSuccess + BackupFile = $LocalBackupFile + } +} + +# Main execution +try { + Write-Log "Starting weekly LIVE backup process" + + # Test prerequisites + if (-not (Test-Prerequisites)) { + throw "Prerequisites check failed" + } + + # Determine which VMs to backup + $VMsToBackup = @() + + if ($AllVMs) { + Write-Log "Backing up all VMs (excluding specified VMs)" + $AllVirtualMachines = Get-VM | Where-Object { $_.Name -notin $ExcludeVMs } + $VMsToBackup = $AllVirtualMachines.Name + + if ($ExcludeVMs.Count -gt 0) { + Write-Log "Excluded VMs: $($ExcludeVMs -join ', ')" + } + Write-Log "VMs to backup: $($VMsToBackup -join ', ')" + } + else { + $VMsToBackup = @($VMName) + Write-Log "Backing up single VM: $VMName" + } + + if ($VMsToBackup.Count -eq 0) { + throw "No VMs found to backup" + } + + # Backup each VM + $SuccessCount = 0 + $FailureCount = 0 + $Results = @() + $SuccessfulBackupFiles = @() + + foreach ($VM in $VMsToBackup) { + $StartTime = Get-Date + $BackupResult = Backup-SingleVM -VMName $VM -MaxBackups $MaxBackups + $EndTime = Get-Date + $Duration = ($EndTime - $StartTime).TotalMinutes + + $Results += [PSCustomObject]@{ + VMName = $VM + Success = $BackupResult.Success + Duration = [math]::Round($Duration, 1) + BackupFile = $BackupResult.BackupFile + } + + if ($BackupResult.Success) { + $SuccessCount++ + if ($BackupResult.BackupFile) { + $SuccessfulBackupFiles += $BackupResult.BackupFile + } + } else { + $FailureCount++ + } + } + + # Copy all successful backups to NAS in one batch + $NASCopySuccess = $false + if ($SuccessfulBackupFiles.Count -gt 0) { + Write-Log "All VM backups completed. Starting bulk copy to NAS..." + $NASCopySuccess = Copy-AllBackupsToNAS -LocalPath $LocalBackupPath -NASPath $NASBackupPath -BackupFiles $SuccessfulBackupFiles + + if ($NASCopySuccess) { + Write-Log "Successfully copied all backups to NAS" + + # Now cleanup old backups on NAS for each VM + Write-Log "Cleaning up old weekly backups on NAS..." + foreach ($VM in $VMsToBackup) { + try { + Cleanup-OldBackups -Path $NASBackupPath -Pattern "^Weekly_$($VM)_\d{8}_\d{4}\.7z$" -Keep $MaxBackups + } + catch { + Write-Log "Warning: Could not cleanup old weekly backups for $VM on NAS: $_" "WARNING" + } + } + } + else { + Write-Log "Failed to copy some or all backups to NAS" "ERROR" + } + } + + # Cleanup old local backups regardless of NAS copy success + Write-Log "Cleaning up old local weekly backups..." + foreach ($VM in $VMsToBackup) { + try { + Cleanup-OldBackups -Path $LocalBackupPath -Pattern "^Weekly_$($VM)_\d{8}_\d{4}\.7z$" -Keep $MaxBackups + } + catch { + Write-Log "Warning: Could not cleanup old local weekly backups for $VM : $_" "WARNING" + } + } + + # Final summary + Write-Log "=== WEEKLY LIVE BACKUP SUMMARY ===" + Write-Log "Total VMs processed: $($VMsToBackup.Count)" + Write-Log "Successful backups: $SuccessCount" + Write-Log "Failed backups: $FailureCount" + Write-Log "NAS copy success: $(if ($NASCopySuccess) { 'YES' } else { 'NO' })" + + foreach ($Result in $Results) { + $Status = if ($Result.Success) { "SUCCESS" } else { "FAILED" } + Write-Log "$($Result.VMName): $Status ($($Result.Duration) minutes)" + } + + # Display local backup status + Write-Log "=== LOCAL WEEKLY BACKUP STATUS ===" + if (Test-Path $LocalBackupPath) { + $LocalBackups = Get-ChildItem -Path $LocalBackupPath -File -Filter "*.7z" | Sort-Object CreationTime -Descending + $TotalLocalSize = 0 + + foreach ($Backup in $LocalBackups) { + $Size = $Backup.Length + $SizeGB = [math]::Round($Size / 1GB, 2) + $TotalLocalSize += $Size + Write-Log "$($Backup.Name): $SizeGB GB (Created: $(Get-Date $Backup.CreationTime -Format 'yyyy-MM-dd HH:mm'))" + } + + $TotalLocalSizeGB = [math]::Round($TotalLocalSize / 1GB, 2) + Write-Log "Total local weekly backup storage used: $TotalLocalSizeGB GB" + } + + if ($FailureCount -gt 0) { + Write-Log "Weekly LIVE backup process completed with $FailureCount failure(s)" "WARNING" + exit 1 + } elseif (-not $NASCopySuccess -and $SuccessfulBackupFiles.Count -gt 0) { + Write-Log "Weekly LIVE backup process completed but NAS copy failed - backups available locally" "WARNING" + exit 1 + } else { + Write-Log "Weekly LIVE backup process completed successfully" "SUCCESS" + } +} +catch { + Write-Log "Weekly LIVE backup process failed: $_" "ERROR" + exit 1 +} + +# Example usage: +# Single VM: +# .\Weekly-VMBackup.ps1 -VMName "MyVM" -MaxBackups 4 +# All VMs: +# .\Weekly-VMBackup.ps1 -AllVMs -MaxBackups 4 +# All VMs except specific ones: +# .\Weekly-VMBackup.ps1 -AllVMs -ExcludeVMs @("TestVM", "TempVM") -MaxBackups 4 + +# Weekly LIVE backup workflow: +# 1. Create checkpoint for consistency (if possible) +# 2. Export running VM to F:\TempBackup\Weekly\{VMName}\Export\ +# 3. Remove checkpoint immediately after export +# 4. Compress with 7-Zip to F:\TempBackup\Weekly\{VMName}\{Archive}.7z (mx=7 for speed) +# 5. Move compressed archive to F:\Backups\Weekly\{Archive}.7z +# 6. Clean up temp files for this VM +# 7. Repeat for all VMs +# 8. Once ALL VMs are done, bulk copy F:\Backups\Weekly\*.7z to NAS +# 9. Clean up old backups on both local F: drive and NAS \ No newline at end of file