diff --git a/.azdo/ci.yml b/.azdo/ci.yml new file mode 100644 index 000000000..1f406777b --- /dev/null +++ b/.azdo/ci.yml @@ -0,0 +1,127 @@ +name: $(BuildDefinitionName)-$(date:yyMM).$(date:dd)$(rev:rrr) +trigger: + # Batch merge builds together while a merge build is running + batch: true + branches: + include: + - latestw_all +pr: + branches: + include: + - latestw_all + +resources: + repositories: + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance + +stages: +- stage: Build + displayName: Build Win32-OpenSSH + jobs: + - job: BuildPkg + displayName: Build Package + pool: + name: 1ES + demands: + - ImageOverride -equals PSMMS2019-OpenSSH-Secure + + steps: + - powershell: | + $powerShellPath = Join-Path -Path $env:AGENT_TEMPDIRECTORY -ChildPath 'powershell' + Invoke-WebRequest -Uri https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/install-powershell.ps1 -outfile ./install-powershell.ps1 + ./install-powershell.ps1 -Destination $powerShellPath + $vstsCommandString = "vso[task.setvariable variable=PATH]$powerShellPath;$env:PATH" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Install PowerShell Core + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + Invoke-AzDOBuild + displayName: Build Win32-OpenSSH + + - pwsh: | + $BuildOutPath = "$(Build.SourcesDirectory)/bin" + $BuildOutx86Path = Join-Path -Path $BuildOutPath -ChildPath 'Win32/Release' + Get-ChildItem -Path $BuildOutx86Path + $BuildOutx64Path = Join-Path -Path $BuildOutPath -ChildPath 'x64/Release' + Get-ChildItem -Path $BuildOutx64Path + displayName: Capture build results + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + $BuildDestPath = "$(Build.SourcesDirectory)/Win32-OpenSSH" + if (Test-Path -Path $BuildDestPath) { + Remove-Item -Path $BuildDestPath -Recurse -Force -ErrorAction SilentlyContinue + } + $null = New-Item -ItemType Directory -Path $BuildDestPath -Force + # Copy build artifacts + $BuildDestx86Path = Join-Path -Path $BuildDestPath -ChildPath 'x86/Release' + Start-OpenSSHPackage -NativeHostArch x86 -Configuration Release -DestinationPath $BuildDestx86Path + $BuildDestX64Path = Join-Path -Path $BuildDestPath -ChildPath 'x64/Release' + Start-OpenSSHPackage -NativeHostArch x64 -Configuration Release -DestinationPath $BuildDestx64Path + # Upload build artifacts + $artifactName = 'Win32-OpenSSH' + Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$BuildDestPath" + displayName: Upload Win32-OpenSSH build artifacts + +#- stage: Compliance +# displayName: Compliance +# dependsOn: Build +# jobs: +# - job: ComplianceJob +# pool: +# vmImage: windows-latest +# steps: +# - checkout: self +# clean: true +# - checkout: ComplianceRepo +# clean: true +# - download: current +# artifact: 'Microsoft.PowerShell.SecretManagement' +# - template: ci-compliance.yml@ComplianceRepo +# parameters: +# # credscan +# suppressionsFile: '' + +- stage: Test + displayName: Test Win32-OpenSSH + dependsOn: Build + jobs: + pool: + vmImage: windows-latest + steps: + - powershell: | + $powerShellPath = Join-Path -Path $env:AGENT_TEMPDIRECTORY -ChildPath 'powershell' + Invoke-WebRequest -Uri https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/install-powershell.ps1 -outfile ./install-powershell.ps1 + ./install-powershell.ps1 -Destination $powerShellPath + $vstsCommandString = "vso[task.setvariable variable=PATH]$powerShellPath;$env:PATH" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Install PowerShell Core + + - task: DownloadBuildArtifacts@0 + displayName: 'Download build artifacts' + inputs: + buildType: current + downloadType: single + artifactName: Win32-OpenSSH + downloadPath: '$(System.ArtifactsDirectory)' + + - pwsh: | + Get-ChildItem -Path "$(System.ArtifactsDirectory)/* -Recurse" + displayName: Capture downloaded artifact directory + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH/x64/Release/*" -OpenSSHDir "$env:SystemDrive/OpenSSH" -NativeHostArch x64 -Configuration Release -Verbose + displayName: Install Win32-OpenSSH + + - pwsh: | + Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force + Invoke-OpenSSHTests -OpenSSHBinPath "$env:SystemDrive/OpenSSH" + Publish-OpenSSHTestResults # TODO: # + displayName: Run tests and publish results diff --git a/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psd1 b/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psd1 new file mode 100644 index 000000000..5eb47018f --- /dev/null +++ b/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psd1 @@ -0,0 +1,37 @@ +## +## Azure DevOps CI build tools +## (TODO: Add appropriate copyright) +## +@{ + +RootModule = './AzDOBuildTools.psm1' + +ModuleVersion = '1.0.0' + +GUID = '0b8fa798-ea71-40c7-b9ab-a417958bb3c4' + +Author = 'Microsoft Corporation' + +CompanyName = 'Microsoft Corporation' + +Copyright = '(c) Microsoft Corporation. All rights reserved.' + +Description = 'AzDO build tools for Win32-OpenSSH repository.' + +PowerShellVersion = '5.1' +DotnetFrameworkVersion = '4.6.1' +CLRVersion = '4.0.0' + +NestedModules = @( + '../OpenSSHCommonUtils.psm1', + '../OpenSSHBuildHelper.psm1', + '../OpenSSHTestHelper.psm1') + +FunctionsToExport = @( + 'Invoke-AllLocally', + 'Invoke-AzDOBuild', + 'Install-OpenSSH', + 'Invoke-OpenSSHTests', + 'Publish-OpenSSHTestResults') + +} diff --git a/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psm1 b/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psm1 new file mode 100644 index 000000000..e278634a5 --- /dev/null +++ b/contrib/win32/openssh/AzDOBuildTools/AzDOBuildTools.psm1 @@ -0,0 +1,449 @@ +## +## Azure DevOps CI build tools +## [Add appropriate copyright] +## + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$repoRoot = Get-RepositoryRoot +$script:messageFile = join-path $repoRoot.FullName "BuildMessage.log" + +function Write-BuildMessage +{ + param ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] $Message, + + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] $Category + ) + + # Write message to verbos stream. + Write-Verbose -Verbose -Message "$Category--$Message" + + # Write it to the log file, if present. + if (-not ([string]::IsNullOrEmpty($script:messageFile))) + { + Add-Content -Path $script:messageFile -Value "$Category--$Message" + } +} + +<# + .Synopsis + Adds a build log to the list of published artifacts. + .Description + If a build log exists, it is renamed to reflect the associated CLR runtime then added to the list of + artifacts to publish. If it doesn't exist, a warning is written and the file is skipped. + The rename is needed since publishing overwrites the artifact if it already exists. + .Parameter artifacts + An array list to add the fully qualified build log path + .Parameter buildLog + The build log file produced by the build. +#> +function Add-BuildLog +{ + param ( + [ValidateNotNull()] + [System.Collections.ArrayList] $artifacts, + + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string] $buildLog + ) + + if (Test-Path -Path $buildLog) + { + $null = $artifacts.Add($buildLog) + } + else + { + Write-Warning "Skip publishing build log. $buildLog does not exist" + } +} + +function Set-BuildVariable +{ + param( + [Parameter(Mandatory=$true)] + [string] + $Name, + + [Parameter(Mandatory=$true)] + [string] + $Value + ) + + Set-Item -Path env:$Name -Value $Value +} + +# Emulates running all of AzDO functions locally. +# This should not be used within an actual AzDO build. +function Invoke-AllLocally +{ + param ( + [switch] $CleanRepo + ) + + if ($CleanRepo) + { + Clear-PSRepo + } + + # TODO: Set up any build environment state here. + + try + { + Invoke-AzDOBuild + Install-OpenSSH + Set-OpenSSHTestEnvironment -confirm:$false + Invoke-OpenSSHTests + Publish-Artifact + } + finally + { + # TODO: Clean up any build environment state here. + } +} + +# Implements the AzDO build package step +function Invoke-AzDOBuild +{ + Set-BuildVariable TestPassed True + Start-OpenSSHBuild -Configuration Release -NativeHostArch x64 -Verbose + Start-OpenSSHBuild -Configuration Release -NativeHostArch x86 -Verbose + Write-BuildMessage -Message "OpenSSH binaries build success!" -Category Information +} + +<# + .Synopsis + Deploy all required files to a location and install the binaries +#> +function Install-OpenSSH +{ + [CmdletBinding()] + param + ( + [Parameter(Mandatory=$true)] + [string]$SourceDir, + + [string]$OpenSSHDir = "$env:SystemDrive\OpenSSH" + ) + + UnInstall-OpenSSH -OpenSSHDir $OpenSSHDir + + if (! (Test-Path -Path $OpenSSHDir)) { + $null = New-Item -Path $OpenSSHDir -ItemType Directory -Force + } + Copy-Item -Path $SourceDir -Destination $OpenSSHDir -Recurse -Force -Verbose + + Push-Location $OpenSSHDir + + try + { + & "$OpenSSHDir\install-sshd.ps1" + + $machinePath = [Environment]::GetEnvironmentVariable('Path', 'MACHINE') + $newMachineEnvironmentPath = $machinePath + if (-not ($machinePath.ToLower().Contains($OpenSSHDir.ToLower()))) + { + $newMachineEnvironmentPath = "$OpenSSHDir;$newMachineEnvironmentPath" + $env:Path = "$OpenSSHDir;$env:Path" + } + + # Update machine environment path + if ($newMachineEnvironmentPath -ne $machinePath) + { + [Environment]::SetEnvironmentVariable('Path', $newMachineEnvironmentPath, 'MACHINE') + } + + Start-Service -Name sshd + Start-Service -Name ssh-agent + } + finally + { + Pop-Location + } + + Write-BuildMessage -Message "OpenSSH installed!" -Category Information +} + +<# + .Synopsis + uninstalled sshd +#> +function UnInstall-OpenSSH +{ + [CmdletBinding()] + param + ( + [string]$OpenSSHDir = "$env:SystemDrive\OpenSSH" + ) + + if (-not (Test-Path $OpenSSHDir -PathType Container)) + { + return + } + + Push-Location $OpenSSHDir + + try + { + if ((Get-Service ssh-agent -ErrorAction SilentlyContinue) -ne $null) { + Stop-Service ssh-agent -Force + } + & "$OpenSSHDir\uninstall-sshd.ps1" + + $machinePath = [Environment]::GetEnvironmentVariable('Path', 'MACHINE') + $newMachineEnvironmentPath = $machinePath + if ($machinePath.ToLower().Contains($OpenSSHDir.ToLower())) + { + $newMachineEnvironmentPath = $newMachineEnvironmentPath.Replace("$OpenSSHDir;", '') + $env:Path = $env:Path.Replace("$OpenSSHDir;", '') + } + + if ($newMachineEnvironmentPath -ne $machinePath) + { + [Environment]::SetEnvironmentVariable('Path', $newMachineEnvironmentPath, 'MACHINE') + } + } + finally + { + Pop-Location + } + + Remove-Item -Path $OpenSSHDir -Recurse -Force -ErrorAction SilentlyContinue +} + +<# + .Synopsis + Publishes package build artifacts. + .Parameter artifacts + An array list to add the fully qualified build log path + .Parameter FileToAdd + Path to the file +#> +function Add-Artifact +{ + param + ( + [ValidateNotNull()] + [System.Collections.ArrayList] $artifacts, + [string] $FileToAdd + ) + + if ([string]::IsNullOrEmpty($FileToAdd) -or (-not (Test-Path $FileToAdd -PathType Leaf)) ) + { + Write-Host "Skip publishing package artifacts. $FileToAdd does not exist" + } + else + { + $null = $artifacts.Add($FileToAdd) + Write-Host "Added $FileToAdd to publishing package artifacts" + } +} + +<# + .Synopsis + After build and test run completes, upload all artifacts from the build machine. +#> +function Publish-Artifact +{ + Write-Host -ForegroundColor Yellow "Publishing project artifacts" + [System.Collections.ArrayList] $artifacts = new-object System.Collections.ArrayList + + # Get the build.log file for each build configuration + Add-BuildLog -artifacts $artifacts -buildLog (Get-BuildLogFile -root $repoRoot.FullName -Configuration Release -NativeHostArch x64) + Add-BuildLog -artifacts $artifacts -buildLog (Get-BuildLogFile -root $repoRoot.FullName -Configuration Release -NativeHostArch x86) + + if($Global:OpenSSHTestInfo) + { + Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["SetupTestResultsFile"] + Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["UnitTestResultsFile"] + Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["E2ETestResultsFile"] + Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["UninstallTestResultsFile"] + Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["TestSetupLogFile"] + } + + if ($Global:bash_tests_summary) + { + Add-Artifact -artifacts $artifacts -FileToAdd $Global:bash_tests_summary["BashTestSummaryFile"] + Add-Artifact -artifacts $artifacts -FileToAdd $Global:bash_tests_summary["BashTestLogFile"] + } + + foreach ($artifact in $artifacts) + { + Write-Host "Publishing $artifact as AzDO artifact" + + # TODO: Create an AzDO artificate upload function. + # Push-AppveyorArtifact $artifact -ErrorAction Continue + } + + Write-Host -ForegroundColor Yellow "End of publishing project artifacts" +} + +<# + .Synopsis + Runs the tests for this repo +#> +function Invoke-OpenSSHTests +{ + [CmdletBinding()] + param ( + [string] $OpenSSHBinPath + ) + + Set-BasicTestInfo -OpenSSHBinPath $OpenSSHBinPath -Confirm:$false + Invoke-OpenSSHSetupTest + if (($OpenSSHTestInfo -eq $null) -or (-not (Test-Path $OpenSSHTestInfo["SetupTestResultsFile"]))) + { + Write-Warning "Test result file $OpenSSHTestInfo["SetupTestResultsFile"] not found after tests." + Write-BuildMessage -Message "Test result file $OpenSSHTestInfo["SetupTestResultsFile"] not found after tests." -Category Error + Set-BuildVariable TestPassed False + Write-Warning "Stop running further tests!" + return + } + $xml = [xml](Get-Content $OpenSSHTestInfo["SetupTestResultsFile"] | out-string) + if ([int]$xml.'test-results'.failures -gt 0) + { + $errorMessage = "$($xml.'test-results'.failures) setup tests in regress\pesterTests failed. Detail test log is at $($OpenSSHTestInfo["SetupTestResultsFile"])." + Write-Warning $errorMessage + Write-BuildMessage -Message $errorMessage -Category Error + Set-BuildVariable TestPassed False + Write-Warning "Stop running further tests!" + return + } + + Write-Host "Start running unit tests" + $unitTestFailed = Invoke-OpenSSHUnitTest + + if($unitTestFailed) + { + Write-Host "At least one of the unit tests failed!" -ForegroundColor Yellow + Write-BuildMessage "At least one of the unit tests failed!" -Category Error + Set-BuildVariable TestPassed False + } + else + { + Write-Host "All Unit tests passed!" + Write-BuildMessage -Message "All Unit tests passed!" -Category Information + } + + # Run all E2E tests. + Set-OpenSSHTestEnvironment -Confirm:$false + Invoke-OpenSSHE2ETest + if (($OpenSSHTestInfo -eq $null) -or (-not (Test-Path $OpenSSHTestInfo["E2ETestResultsFile"]))) + { + Write-Warning "Test result file $OpenSSHTestInfo["E2ETestResultsFile"] not found after tests." + Write-BuildMessage -Message "Test result file $OpenSSHTestInfo["E2ETestResultsFile"] not found after tests." -Category Error + Set-BuildVariable TestPassed False + Write-Warning "Stop running further tests!" + return + } + $xml = [xml](Get-Content $OpenSSHTestInfo["E2ETestResultsFile"] | out-string) + if ([int]$xml.'test-results'.failures -gt 0) + { + $errorMessage = "$($xml.'test-results'.failures) tests in regress\pesterTests failed. Detail test log is at $($OpenSSHTestInfo["E2ETestResultsFile"])." + Write-Warning $errorMessage + Write-BuildMessage -Message $errorMessage -Category Error + Set-BuildVariable TestPassed False + Write-Warning "Stop running further tests!" + return + } + + # Run UNIX bash tests. + Invoke-OpenSSHBashTests + if (-not $Global:bash_tests_summary) + { + $errorMessage = "Failed to start OpenSSH bash tests" + Write-Warning $errorMessage + Write-BuildMessage -Message $errorMessage -Category Error + Set-BuildVariable TestPassed False + Write-Warning "Stop running further tests!" + return + } + + if ($Global:bash_tests_summary["TotalBashTestsFailed"] -ne 0) + { + $total_bash_failed_tests = $Global:bash_tests_summary["TotalBashTestsFailed"] + $total_bash_tests = $Global:bash_tests_summary["TotalBashTests"] + $errorMessage = "At least one of the bash tests failed. [$total_bash_failed_tests of $total_bash_tests]" + Write-Warning $errorMessage + Write-BuildMessage -Message $errorMessage -Category Error + Set-BuildVariable TestPassed False + Write-Warning "Stop running further tests!" + return + } + + Invoke-OpenSSHUninstallTest + if (($OpenSSHTestInfo -eq $null) -or (-not (Test-Path $OpenSSHTestInfo["UninstallTestResultsFile"]))) + { + Write-Warning "Test result file $OpenSSHTestInfo["UninstallTestResultsFile"] not found after tests." + Write-BuildMessage -Message "Test result file $OpenSSHTestInfo["UninstallTestResultsFile"] not found after tests." -Category Error + Set-BuildVariable TestPassed False + } + else { + $xml = [xml](Get-Content $OpenSSHTestInfo["UninstallTestResultsFile"] | out-string) + if ([int]$xml.'test-results'.failures -gt 0) + { + $errorMessage = "$($xml.'test-results'.failures) uninstall tests in regress\pesterTests failed. Detail test log is at $($OpenSSHTestInfo["UninstallTestResultsFile"])." + Write-Warning $errorMessage + Write-BuildMessage -Message $errorMessage -Category Error + Set-BuildVariable TestPassed False + } + } + + # Writing out warning when the $Error.Count is non-zero. Tests Should clean $Error after success. + if ($Error.Count -gt 0) + { + Write-BuildMessage -Message "Tests Should clean $Error after success." -Category Warning + } +} + +<# + .Synopsis + upload OpenSSH pester test results. +#> +function Publish-OpenSSHTestResults +{ + if ($env:APPVEYOR_JOB_ID) + { + $setupresultFile = Resolve-Path $Global:OpenSSHTestInfo["SetupTestResultsFile"] -ErrorAction Ignore + if( (Test-Path $Global:OpenSSHTestInfo["SetupTestResultsFile"]) -and $setupresultFile) + { + (New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", $setupresultFile) + Write-BuildMessage -Message "Setup test results uploaded!" -Category Information + } + + $E2EresultFile = Resolve-Path $Global:OpenSSHTestInfo["E2ETestResultsFile"] -ErrorAction Ignore + if( (Test-Path $Global:OpenSSHTestInfo["E2ETestResultsFile"]) -and $E2EresultFile) + { + (New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", $E2EresultFile) + Write-BuildMessage -Message "E2E test results uploaded!" -Category Information + } + + $uninstallResultFile = Resolve-Path $Global:OpenSSHTestInfo["UninstallTestResultsFile"] -ErrorAction Ignore + if( (Test-Path $Global:OpenSSHTestInfo["UninstallTestResultsFile"]) -and $uninstallResultFile) + { + (New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", $uninstallResultFile) + Write-BuildMessage -Message "Uninstall test results uploaded!" -Category Information + } + } + + if ($env:DebugMode) + { + Remove-Item $env:DebugMode + } + + if($env:TestPassed -ieq 'True') + { + Write-BuildMessage -Message "The checkin validation success!" -Category Information + } + else + { + Write-BuildMessage -Message "The checkin validation failed!" -Category Error + throw "The checkin validation failed!" + } +} diff --git a/contrib/win32/openssh/OpenSSHBuildHelper.psm1 b/contrib/win32/openssh/OpenSSHBuildHelper.psm1 index a152542d2..732ac1985 100644 --- a/contrib/win32/openssh/OpenSSHBuildHelper.psm1 +++ b/contrib/win32/openssh/OpenSSHBuildHelper.psm1 @@ -567,7 +567,7 @@ function Start-OpenSSHBuild Start-OpenSSHBootstrap -NativeHostArch $NativeHostArch -OneCore:$OneCore - $PathTargets = Join-Path $PSScriptRoot paths.targets + $PathTargets = Join-Path -Path $PSScriptRoot -ChildPath 'paths.targets' if ($NoOpenSSL) { [XML]$xml = Get-Content $PathTargets