Add yaml and test files to support AzDO CI (#629)

* Initial changes for AzDO

* Implement build and artifact upload
This commit is contained in:
Paul Higinbotham 2022-11-02 12:52:10 -07:00 committed by GitHub
parent 316ebdca0c
commit d9738a3f13
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 614 additions and 1 deletions

127
.azdo/ci.yml Normal file
View File

@ -0,0 +1,127 @@
name: $(BuildDefinitionName)-$(date:yyMM).$(date:dd)$(rev:rrr)
trigger:
# Batch merge builds together while a merge build is running
batch: true
branches:
include:
- latestw_all
pr:
branches:
include:
- latestw_all
resources:
repositories:
- repository: ComplianceRepo
type: github
endpoint: ComplianceGHRepo
name: PowerShell/compliance
stages:
- stage: Build
displayName: Build Win32-OpenSSH
jobs:
- job: BuildPkg
displayName: Build Package
pool:
name: 1ES
demands:
- ImageOverride -equals PSMMS2019-OpenSSH-Secure
steps:
- powershell: |
$powerShellPath = Join-Path -Path $env:AGENT_TEMPDIRECTORY -ChildPath 'powershell'
Invoke-WebRequest -Uri https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/install-powershell.ps1 -outfile ./install-powershell.ps1
./install-powershell.ps1 -Destination $powerShellPath
$vstsCommandString = "vso[task.setvariable variable=PATH]$powerShellPath;$env:PATH"
Write-Host "sending " + $vstsCommandString
Write-Host "##$vstsCommandString"
displayName: Install PowerShell Core
- pwsh: |
Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
Invoke-AzDOBuild
displayName: Build Win32-OpenSSH
- pwsh: |
$BuildOutPath = "$(Build.SourcesDirectory)/bin"
$BuildOutx86Path = Join-Path -Path $BuildOutPath -ChildPath 'Win32/Release'
Get-ChildItem -Path $BuildOutx86Path
$BuildOutx64Path = Join-Path -Path $BuildOutPath -ChildPath 'x64/Release'
Get-ChildItem -Path $BuildOutx64Path
displayName: Capture build results
- pwsh: |
Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
$BuildDestPath = "$(Build.SourcesDirectory)/Win32-OpenSSH"
if (Test-Path -Path $BuildDestPath) {
Remove-Item -Path $BuildDestPath -Recurse -Force -ErrorAction SilentlyContinue
}
$null = New-Item -ItemType Directory -Path $BuildDestPath -Force
# Copy build artifacts
$BuildDestx86Path = Join-Path -Path $BuildDestPath -ChildPath 'x86/Release'
Start-OpenSSHPackage -NativeHostArch x86 -Configuration Release -DestinationPath $BuildDestx86Path
$BuildDestX64Path = Join-Path -Path $BuildDestPath -ChildPath 'x64/Release'
Start-OpenSSHPackage -NativeHostArch x64 -Configuration Release -DestinationPath $BuildDestx64Path
# Upload build artifacts
$artifactName = 'Win32-OpenSSH'
Write-Host "##vso[artifact.upload containerfolder=$artifactName;artifactname=$artifactName;]$BuildDestPath"
displayName: Upload Win32-OpenSSH build artifacts
#- stage: Compliance
# displayName: Compliance
# dependsOn: Build
# jobs:
# - job: ComplianceJob
# pool:
# vmImage: windows-latest
# steps:
# - checkout: self
# clean: true
# - checkout: ComplianceRepo
# clean: true
# - download: current
# artifact: 'Microsoft.PowerShell.SecretManagement'
# - template: ci-compliance.yml@ComplianceRepo
# parameters:
# # credscan
# suppressionsFile: ''
- stage: Test
displayName: Test Win32-OpenSSH
dependsOn: Build
jobs:
pool:
vmImage: windows-latest
steps:
- powershell: |
$powerShellPath = Join-Path -Path $env:AGENT_TEMPDIRECTORY -ChildPath 'powershell'
Invoke-WebRequest -Uri https://raw.githubusercontent.com/PowerShell/PowerShell/master/tools/install-powershell.ps1 -outfile ./install-powershell.ps1
./install-powershell.ps1 -Destination $powerShellPath
$vstsCommandString = "vso[task.setvariable variable=PATH]$powerShellPath;$env:PATH"
Write-Host "sending " + $vstsCommandString
Write-Host "##$vstsCommandString"
displayName: Install PowerShell Core
- task: DownloadBuildArtifacts@0
displayName: 'Download build artifacts'
inputs:
buildType: current
downloadType: single
artifactName: Win32-OpenSSH
downloadPath: '$(System.ArtifactsDirectory)'
- pwsh: |
Get-ChildItem -Path "$(System.ArtifactsDirectory)/* -Recurse"
displayName: Capture downloaded artifact directory
- pwsh: |
Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
Install-OpenSSH -SourceDir "$(System.ArtifactsDirectory)/Win32-OpenSSH/x64/Release/*" -OpenSSHDir "$env:SystemDrive/OpenSSH" -NativeHostArch x64 -Configuration Release -Verbose
displayName: Install Win32-OpenSSH
- pwsh: |
Import-Module -Name "$(Build.SourcesDirectory)/contrib/win32/openssh/AzDOBuildTools" -Force
Invoke-OpenSSHTests -OpenSSHBinPath "$env:SystemDrive/OpenSSH"
Publish-OpenSSHTestResults # TODO: #
displayName: Run tests and publish results

View File

@ -0,0 +1,37 @@
##
## Azure DevOps CI build tools
## (TODO: Add appropriate copyright)
##
@{
RootModule = './AzDOBuildTools.psm1'
ModuleVersion = '1.0.0'
GUID = '0b8fa798-ea71-40c7-b9ab-a417958bb3c4'
Author = 'Microsoft Corporation'
CompanyName = 'Microsoft Corporation'
Copyright = '(c) Microsoft Corporation. All rights reserved.'
Description = 'AzDO build tools for Win32-OpenSSH repository.'
PowerShellVersion = '5.1'
DotnetFrameworkVersion = '4.6.1'
CLRVersion = '4.0.0'
NestedModules = @(
'../OpenSSHCommonUtils.psm1',
'../OpenSSHBuildHelper.psm1',
'../OpenSSHTestHelper.psm1')
FunctionsToExport = @(
'Invoke-AllLocally',
'Invoke-AzDOBuild',
'Install-OpenSSH',
'Invoke-OpenSSHTests',
'Publish-OpenSSHTestResults')
}

View File

@ -0,0 +1,449 @@
##
## Azure DevOps CI build tools
## [Add appropriate copyright]
##
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$repoRoot = Get-RepositoryRoot
$script:messageFile = join-path $repoRoot.FullName "BuildMessage.log"
function Write-BuildMessage
{
param (
[Parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[string] $Message,
[Parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[string] $Category
)
# Write message to verbos stream.
Write-Verbose -Verbose -Message "$Category--$Message"
# Write it to the log file, if present.
if (-not ([string]::IsNullOrEmpty($script:messageFile)))
{
Add-Content -Path $script:messageFile -Value "$Category--$Message"
}
}
<#
.Synopsis
Adds a build log to the list of published artifacts.
.Description
If a build log exists, it is renamed to reflect the associated CLR runtime then added to the list of
artifacts to publish. If it doesn't exist, a warning is written and the file is skipped.
The rename is needed since publishing overwrites the artifact if it already exists.
.Parameter artifacts
An array list to add the fully qualified build log path
.Parameter buildLog
The build log file produced by the build.
#>
function Add-BuildLog
{
param (
[ValidateNotNull()]
[System.Collections.ArrayList] $artifacts,
[Parameter(Mandatory=$true)]
[ValidateNotNullOrEmpty()]
[string] $buildLog
)
if (Test-Path -Path $buildLog)
{
$null = $artifacts.Add($buildLog)
}
else
{
Write-Warning "Skip publishing build log. $buildLog does not exist"
}
}
function Set-BuildVariable
{
param(
[Parameter(Mandatory=$true)]
[string]
$Name,
[Parameter(Mandatory=$true)]
[string]
$Value
)
Set-Item -Path env:$Name -Value $Value
}
# Emulates running all of AzDO functions locally.
# This should not be used within an actual AzDO build.
function Invoke-AllLocally
{
param (
[switch] $CleanRepo
)
if ($CleanRepo)
{
Clear-PSRepo
}
# TODO: Set up any build environment state here.
try
{
Invoke-AzDOBuild
Install-OpenSSH
Set-OpenSSHTestEnvironment -confirm:$false
Invoke-OpenSSHTests
Publish-Artifact
}
finally
{
# TODO: Clean up any build environment state here.
}
}
# Implements the AzDO build package step
function Invoke-AzDOBuild
{
Set-BuildVariable TestPassed True
Start-OpenSSHBuild -Configuration Release -NativeHostArch x64 -Verbose
Start-OpenSSHBuild -Configuration Release -NativeHostArch x86 -Verbose
Write-BuildMessage -Message "OpenSSH binaries build success!" -Category Information
}
<#
.Synopsis
Deploy all required files to a location and install the binaries
#>
function Install-OpenSSH
{
[CmdletBinding()]
param
(
[Parameter(Mandatory=$true)]
[string]$SourceDir,
[string]$OpenSSHDir = "$env:SystemDrive\OpenSSH"
)
UnInstall-OpenSSH -OpenSSHDir $OpenSSHDir
if (! (Test-Path -Path $OpenSSHDir)) {
$null = New-Item -Path $OpenSSHDir -ItemType Directory -Force
}
Copy-Item -Path $SourceDir -Destination $OpenSSHDir -Recurse -Force -Verbose
Push-Location $OpenSSHDir
try
{
& "$OpenSSHDir\install-sshd.ps1"
$machinePath = [Environment]::GetEnvironmentVariable('Path', 'MACHINE')
$newMachineEnvironmentPath = $machinePath
if (-not ($machinePath.ToLower().Contains($OpenSSHDir.ToLower())))
{
$newMachineEnvironmentPath = "$OpenSSHDir;$newMachineEnvironmentPath"
$env:Path = "$OpenSSHDir;$env:Path"
}
# Update machine environment path
if ($newMachineEnvironmentPath -ne $machinePath)
{
[Environment]::SetEnvironmentVariable('Path', $newMachineEnvironmentPath, 'MACHINE')
}
Start-Service -Name sshd
Start-Service -Name ssh-agent
}
finally
{
Pop-Location
}
Write-BuildMessage -Message "OpenSSH installed!" -Category Information
}
<#
.Synopsis
uninstalled sshd
#>
function UnInstall-OpenSSH
{
[CmdletBinding()]
param
(
[string]$OpenSSHDir = "$env:SystemDrive\OpenSSH"
)
if (-not (Test-Path $OpenSSHDir -PathType Container))
{
return
}
Push-Location $OpenSSHDir
try
{
if ((Get-Service ssh-agent -ErrorAction SilentlyContinue) -ne $null) {
Stop-Service ssh-agent -Force
}
& "$OpenSSHDir\uninstall-sshd.ps1"
$machinePath = [Environment]::GetEnvironmentVariable('Path', 'MACHINE')
$newMachineEnvironmentPath = $machinePath
if ($machinePath.ToLower().Contains($OpenSSHDir.ToLower()))
{
$newMachineEnvironmentPath = $newMachineEnvironmentPath.Replace("$OpenSSHDir;", '')
$env:Path = $env:Path.Replace("$OpenSSHDir;", '')
}
if ($newMachineEnvironmentPath -ne $machinePath)
{
[Environment]::SetEnvironmentVariable('Path', $newMachineEnvironmentPath, 'MACHINE')
}
}
finally
{
Pop-Location
}
Remove-Item -Path $OpenSSHDir -Recurse -Force -ErrorAction SilentlyContinue
}
<#
.Synopsis
Publishes package build artifacts.
.Parameter artifacts
An array list to add the fully qualified build log path
.Parameter FileToAdd
Path to the file
#>
function Add-Artifact
{
param
(
[ValidateNotNull()]
[System.Collections.ArrayList] $artifacts,
[string] $FileToAdd
)
if ([string]::IsNullOrEmpty($FileToAdd) -or (-not (Test-Path $FileToAdd -PathType Leaf)) )
{
Write-Host "Skip publishing package artifacts. $FileToAdd does not exist"
}
else
{
$null = $artifacts.Add($FileToAdd)
Write-Host "Added $FileToAdd to publishing package artifacts"
}
}
<#
.Synopsis
After build and test run completes, upload all artifacts from the build machine.
#>
function Publish-Artifact
{
Write-Host -ForegroundColor Yellow "Publishing project artifacts"
[System.Collections.ArrayList] $artifacts = new-object System.Collections.ArrayList
# Get the build.log file for each build configuration
Add-BuildLog -artifacts $artifacts -buildLog (Get-BuildLogFile -root $repoRoot.FullName -Configuration Release -NativeHostArch x64)
Add-BuildLog -artifacts $artifacts -buildLog (Get-BuildLogFile -root $repoRoot.FullName -Configuration Release -NativeHostArch x86)
if($Global:OpenSSHTestInfo)
{
Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["SetupTestResultsFile"]
Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["UnitTestResultsFile"]
Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["E2ETestResultsFile"]
Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["UninstallTestResultsFile"]
Add-Artifact -artifacts $artifacts -FileToAdd $Global:OpenSSHTestInfo["TestSetupLogFile"]
}
if ($Global:bash_tests_summary)
{
Add-Artifact -artifacts $artifacts -FileToAdd $Global:bash_tests_summary["BashTestSummaryFile"]
Add-Artifact -artifacts $artifacts -FileToAdd $Global:bash_tests_summary["BashTestLogFile"]
}
foreach ($artifact in $artifacts)
{
Write-Host "Publishing $artifact as AzDO artifact"
# TODO: Create an AzDO artificate upload function.
# Push-AppveyorArtifact $artifact -ErrorAction Continue
}
Write-Host -ForegroundColor Yellow "End of publishing project artifacts"
}
<#
.Synopsis
Runs the tests for this repo
#>
function Invoke-OpenSSHTests
{
[CmdletBinding()]
param (
[string] $OpenSSHBinPath
)
Set-BasicTestInfo -OpenSSHBinPath $OpenSSHBinPath -Confirm:$false
Invoke-OpenSSHSetupTest
if (($OpenSSHTestInfo -eq $null) -or (-not (Test-Path $OpenSSHTestInfo["SetupTestResultsFile"])))
{
Write-Warning "Test result file $OpenSSHTestInfo["SetupTestResultsFile"] not found after tests."
Write-BuildMessage -Message "Test result file $OpenSSHTestInfo["SetupTestResultsFile"] not found after tests." -Category Error
Set-BuildVariable TestPassed False
Write-Warning "Stop running further tests!"
return
}
$xml = [xml](Get-Content $OpenSSHTestInfo["SetupTestResultsFile"] | out-string)
if ([int]$xml.'test-results'.failures -gt 0)
{
$errorMessage = "$($xml.'test-results'.failures) setup tests in regress\pesterTests failed. Detail test log is at $($OpenSSHTestInfo["SetupTestResultsFile"])."
Write-Warning $errorMessage
Write-BuildMessage -Message $errorMessage -Category Error
Set-BuildVariable TestPassed False
Write-Warning "Stop running further tests!"
return
}
Write-Host "Start running unit tests"
$unitTestFailed = Invoke-OpenSSHUnitTest
if($unitTestFailed)
{
Write-Host "At least one of the unit tests failed!" -ForegroundColor Yellow
Write-BuildMessage "At least one of the unit tests failed!" -Category Error
Set-BuildVariable TestPassed False
}
else
{
Write-Host "All Unit tests passed!"
Write-BuildMessage -Message "All Unit tests passed!" -Category Information
}
# Run all E2E tests.
Set-OpenSSHTestEnvironment -Confirm:$false
Invoke-OpenSSHE2ETest
if (($OpenSSHTestInfo -eq $null) -or (-not (Test-Path $OpenSSHTestInfo["E2ETestResultsFile"])))
{
Write-Warning "Test result file $OpenSSHTestInfo["E2ETestResultsFile"] not found after tests."
Write-BuildMessage -Message "Test result file $OpenSSHTestInfo["E2ETestResultsFile"] not found after tests." -Category Error
Set-BuildVariable TestPassed False
Write-Warning "Stop running further tests!"
return
}
$xml = [xml](Get-Content $OpenSSHTestInfo["E2ETestResultsFile"] | out-string)
if ([int]$xml.'test-results'.failures -gt 0)
{
$errorMessage = "$($xml.'test-results'.failures) tests in regress\pesterTests failed. Detail test log is at $($OpenSSHTestInfo["E2ETestResultsFile"])."
Write-Warning $errorMessage
Write-BuildMessage -Message $errorMessage -Category Error
Set-BuildVariable TestPassed False
Write-Warning "Stop running further tests!"
return
}
# Run UNIX bash tests.
Invoke-OpenSSHBashTests
if (-not $Global:bash_tests_summary)
{
$errorMessage = "Failed to start OpenSSH bash tests"
Write-Warning $errorMessage
Write-BuildMessage -Message $errorMessage -Category Error
Set-BuildVariable TestPassed False
Write-Warning "Stop running further tests!"
return
}
if ($Global:bash_tests_summary["TotalBashTestsFailed"] -ne 0)
{
$total_bash_failed_tests = $Global:bash_tests_summary["TotalBashTestsFailed"]
$total_bash_tests = $Global:bash_tests_summary["TotalBashTests"]
$errorMessage = "At least one of the bash tests failed. [$total_bash_failed_tests of $total_bash_tests]"
Write-Warning $errorMessage
Write-BuildMessage -Message $errorMessage -Category Error
Set-BuildVariable TestPassed False
Write-Warning "Stop running further tests!"
return
}
Invoke-OpenSSHUninstallTest
if (($OpenSSHTestInfo -eq $null) -or (-not (Test-Path $OpenSSHTestInfo["UninstallTestResultsFile"])))
{
Write-Warning "Test result file $OpenSSHTestInfo["UninstallTestResultsFile"] not found after tests."
Write-BuildMessage -Message "Test result file $OpenSSHTestInfo["UninstallTestResultsFile"] not found after tests." -Category Error
Set-BuildVariable TestPassed False
}
else {
$xml = [xml](Get-Content $OpenSSHTestInfo["UninstallTestResultsFile"] | out-string)
if ([int]$xml.'test-results'.failures -gt 0)
{
$errorMessage = "$($xml.'test-results'.failures) uninstall tests in regress\pesterTests failed. Detail test log is at $($OpenSSHTestInfo["UninstallTestResultsFile"])."
Write-Warning $errorMessage
Write-BuildMessage -Message $errorMessage -Category Error
Set-BuildVariable TestPassed False
}
}
# Writing out warning when the $Error.Count is non-zero. Tests Should clean $Error after success.
if ($Error.Count -gt 0)
{
Write-BuildMessage -Message "Tests Should clean $Error after success." -Category Warning
}
}
<#
.Synopsis
upload OpenSSH pester test results.
#>
function Publish-OpenSSHTestResults
{
if ($env:APPVEYOR_JOB_ID)
{
$setupresultFile = Resolve-Path $Global:OpenSSHTestInfo["SetupTestResultsFile"] -ErrorAction Ignore
if( (Test-Path $Global:OpenSSHTestInfo["SetupTestResultsFile"]) -and $setupresultFile)
{
(New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", $setupresultFile)
Write-BuildMessage -Message "Setup test results uploaded!" -Category Information
}
$E2EresultFile = Resolve-Path $Global:OpenSSHTestInfo["E2ETestResultsFile"] -ErrorAction Ignore
if( (Test-Path $Global:OpenSSHTestInfo["E2ETestResultsFile"]) -and $E2EresultFile)
{
(New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", $E2EresultFile)
Write-BuildMessage -Message "E2E test results uploaded!" -Category Information
}
$uninstallResultFile = Resolve-Path $Global:OpenSSHTestInfo["UninstallTestResultsFile"] -ErrorAction Ignore
if( (Test-Path $Global:OpenSSHTestInfo["UninstallTestResultsFile"]) -and $uninstallResultFile)
{
(New-Object 'System.Net.WebClient').UploadFile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", $uninstallResultFile)
Write-BuildMessage -Message "Uninstall test results uploaded!" -Category Information
}
}
if ($env:DebugMode)
{
Remove-Item $env:DebugMode
}
if($env:TestPassed -ieq 'True')
{
Write-BuildMessage -Message "The checkin validation success!" -Category Information
}
else
{
Write-BuildMessage -Message "The checkin validation failed!" -Category Error
throw "The checkin validation failed!"
}
}

View File

@ -567,7 +567,7 @@ function Start-OpenSSHBuild
Start-OpenSSHBootstrap -NativeHostArch $NativeHostArch -OneCore:$OneCore
$PathTargets = Join-Path $PSScriptRoot paths.targets
$PathTargets = Join-Path -Path $PSScriptRoot -ChildPath 'paths.targets'
if ($NoOpenSSL)
{
[XML]$xml = Get-Content $PathTargets