diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml new file mode 100644 index 000000000..56b7b3aaf --- /dev/null +++ b/.github/workflows/ci-script-tests.yml @@ -0,0 +1,42 @@ +name: CI script tests + +# Runs the Pester tests of the scripts in ci on Linux and Windows, where the +# builds run them. The environment variable names workflow runs its own test. + +on: + pull_request: + paths: + - 'ci/**' + - '.github/workflows/ci-script-tests.yml' + push: + branches: + - main + - rc + paths: + - 'ci/**' + - '.github/workflows/ci-script-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + ci-script-tests: + name: Test the CI scripts on ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Run the tests + shell: pwsh + run: | + $tests = Get-ChildItem ./ci -Filter *.Tests.ps1 | + Where-Object Name -ne 'check-environment-variable-names.Tests.ps1' + Invoke-Pester -Path $tests.FullName -CI -Output Detailed diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index e99cdcb13..eee03edfd 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -29,7 +29,7 @@ on: schedule: - cron: '10 5 * * *' pull_request: - branches: [main] + branches: [main, rc] paths: - 'pipeline/pipeline.did/**' - 'pipeline/owid-java/**' diff --git a/.github/workflows/nightly-pipeline.yml b/.github/workflows/nightly-pipeline.yml index ce793b713..54114ab6c 100644 --- a/.github/workflows/nightly-pipeline.yml +++ b/.github/workflows/nightly-pipeline.yml @@ -13,6 +13,35 @@ on: - cron: '0 3 * * *' jobs: + # The publish builds main, which the pull requests into rc do not change, so + # it runs first and has the runners before the pull request builds do. + Publish: + name: Nightly Publish + uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main + with: + repo-name: ${{ github.event.repository.name }} + org-name: ${{ github.event.repository.owner.login }} + dryrun: ${{ inputs.dryrun || false }} + common-ci-ref: ${{ inputs.common-ci-ref || '' }} + cache-assets: true + secrets: + token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + MavenSettings: ${{ secrets.MAVEN_SETTINGS }} + JavaGpgKeyPassphrase: ${{ secrets.JAVA_GPG_KEY_PASSPHRASE }} + JavaPGP: ${{ secrets.JAVA_KEY_PGP_FILE }} + CodeSigningKeyVaultName: ${{ secrets.CODE_SIGNING_KEY_VAULT_NAME }} + CodeSigningKeyVaultUrl: ${{ secrets.CODE_SIGNING_KEY_VAULT_URL }} + CodeSigningKeyVaultClientId: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_ID }} + CodeSigningKeyVaultTenantId: ${{ secrets.CODE_SIGNING_KEY_VAULT_TENANT_ID }} + CodeSigningKeyVaultClientSecret: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_SECRET }} + CodeSigningKeyVaultCertificateName: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_NAME }} + CodeSigningKeyVaultCertificateData: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_DATA }} + DataFileChange: name: Nightly Data File Change uses: 51Degrees/common-ci/.github/workflows/nightly-data-file-change.yml@main @@ -50,7 +79,7 @@ jobs: PullRequests: if: ${{ !cancelled() }} - needs: [DataFileChange, PackageUpdate, SubmoduleUpdate] + needs: [Publish, DataFileChange, PackageUpdate, SubmoduleUpdate] name: Nightly Pull Requests uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main with: @@ -59,6 +88,8 @@ jobs: dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} cache-assets: true + # One configuration that fails does not cancel the others. + fail-fast: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} @@ -66,32 +97,3 @@ jobs: IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - - Publish: - if: ${{ !cancelled() }} - needs: PullRequests - name: Nightly Publish - uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - common-ci-ref: ${{ inputs.common-ci-ref || '' }} - cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - MavenSettings: ${{ secrets.MAVEN_SETTINGS }} - JavaGpgKeyPassphrase: ${{ secrets.JAVA_GPG_KEY_PASSPHRASE }} - JavaPGP: ${{ secrets.JAVA_KEY_PGP_FILE }} - CodeSigningKeyVaultName: ${{ secrets.CODE_SIGNING_KEY_VAULT_NAME }} - CodeSigningKeyVaultUrl: ${{ secrets.CODE_SIGNING_KEY_VAULT_URL }} - CodeSigningKeyVaultClientId: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_ID }} - CodeSigningKeyVaultTenantId: ${{ secrets.CODE_SIGNING_KEY_VAULT_TENANT_ID }} - CodeSigningKeyVaultClientSecret: ${{ secrets.CODE_SIGNING_KEY_VAULT_CLIENT_SECRET }} - CodeSigningKeyVaultCertificateName: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_NAME }} - CodeSigningKeyVaultCertificateData: ${{ secrets.CODE_SIGNING_KEY_VAULT_CERTIFICATE_DATA }} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index c1fda910b..5a8462654 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -9,6 +9,10 @@ on: common-ci-ref: type: string default: '' + full-matrix: + description: 'Build every configuration, not only those the changes need' + type: boolean + default: false jobs: PullRequests: @@ -20,6 +24,12 @@ jobs: dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} cache-assets: true + # Each pull request is built on the configurations its changes need, + # as ci/select-options.ps1 chooses, and one that fails does not cancel + # the others. With nothing eligible, the nightly builds rc instead. + full-matrix: ${{ inputs.full-matrix || false }} + fail-fast: false + build-without-pull-requests: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} diff --git a/.github/workflows/release-pull-request.yml b/.github/workflows/release-pull-request.yml new file mode 100644 index 000000000..4dfeb08f9 --- /dev/null +++ b/.github/workflows/release-pull-request.yml @@ -0,0 +1,31 @@ +name: Release Pull Request + +# Builds and tests the release pull request, rc into main, on every entry in +# ci/options.json, and never merges it. A pull request into rc is built only on +# the entries its changes need, so this is the full build of a release before +# it reaches main. See common-ci .github/workflows/release-pull-request.yml. +on: + pull_request: + branches: [main] + +# A push to rc while the release is open builds it again, in place of the +# build of the older rc. +concurrency: + group: release-pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + Release: + name: Release Pull Request + uses: 51Degrees/common-ci/.github/workflows/release-pull-request.yml@main + with: + repo-name: ${{ github.event.repository.name }} + org-name: ${{ github.event.repository.owner.login }} + cache-assets: true + secrets: + token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} diff --git a/ci/README.md b/ci/README.md index b00a5034a..9b205e57f 100644 --- a/ci/README.md +++ b/ci/README.md @@ -20,6 +20,18 @@ repositories used to do for one package tree each. request tests the examples against the libraries it changes. The publish workflow instead tests the examples against the version it built, through [run-integration-tests.ps1](run-integration-tests.ps1). +- [select-options.ps1](select-options.ps1) chooses the entries a pull request + runs. The standard set is JDK 17 and JDK 8 on Linux, without the three IP + Intelligence tests in [standard-excludes.txt](standard-excludes.txt) and + with the Selenium contract on JDK 17 alone. A change to a native module, IP + Intelligence, a `pom.xml` or a file outside the package trees adds to it. + macOS and performance run only when `rc` itself or the release pull request + is built, which + [release-pull-request.yml](../.github/workflows/release-pull-request.yml) + does on every entry. The Pester 5 tests of select-options.ps1 are + [select-options.Tests.ps1](select-options.Tests.ps1), which + [ci-script-tests.yml](../.github/workflows/ci-script-tests.yml) runs on + Linux and Windows for each pull request that changes this folder. - [fetch-assets.ps1](fetch-assets.ps1) fetches every data file as one of common-ci's assets, which each run caches once, and links or copies it into `cxx/data/device-detection` or `cxx/data/ip-intelligence`, where the tests diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index ff9175676..7f267eff1 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -7,7 +7,10 @@ param( [string]$TestResourceKey, [string]$DeviceDetection, [string]$IpIntelligence, - [string]$IpIntelligenceUrl + [string]$IpIntelligenceUrl, + # False, from select-options.ps1, on every entry of a pull request but + # one, so the browser contract runs once. + [bool]$SeleniumContract = $true ) $ErrorActionPreference = "Stop" # Each part's exit code is checked below, so one failing part does not stop @@ -131,7 +134,9 @@ function Invoke-ContractTests { } Write-Host "::group::Selenium contract tests" -if (-not $TestResourceKey) { +if (-not $SeleniumContract) { + Write-Host "Another entry runs the Selenium contract tests for this pull request." +} elseif (-not $TestResourceKey) { Write-Host "::warning title=Selenium skipped::No resource key, so the Selenium contract tests were not run against the web examples." } elseif ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture -eq [System.Runtime.InteropServices.Architecture]::Arm64) { Write-Host "::warning title=Selenium skipped::Selenium contract skipped on linux-arm64, which has no selenium-manager build. It runs on x64 and macOS arm64." diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index a54755337..c905c0912 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -2,7 +2,11 @@ param( [Parameter(Mandatory)][string]$RepoName, [string]$Name, [string]$TestResourceKey, - [string]$DeviceDetection + [string]$DeviceDetection, + # How deeply a pull request is tested, as select-options.ps1 chose. + # standard leaves out the tests listed in standard-excludes.txt. Empty, for + # the branch itself or a release, runs every test. + [string]$Depth ) $ErrorActionPreference = "Stop" @@ -13,12 +17,17 @@ $ErrorActionPreference = "Stop" # does. The dev profile points the examples at the reactor. SuperResourceKey is # read by the Location and Device Detection examples tests, and LicenseKey by # the Device Detection data file update example. The IP Intelligence update -# example's test is ignored, so it does not need a key of its own. -./java/run-unit-tests.ps1 -RepoName $RepoName -Name $Name -ExtraArgs @( +# example's test is ignored, so it does not need a key of its own. A pull +# request at the standard depth leaves out the tests in standard-excludes.txt. +$extraArgs = @( '-Pdev' "-DTestResourceKey=$TestResourceKey" "-DSuperResourceKey=$TestResourceKey" "-DLicenseKey=$DeviceDetection" ) +if ($Depth -eq 'standard') { + $extraArgs += "-Dsurefire.excludesFile=$PSScriptRoot/standard-excludes.txt" +} +./java/run-unit-tests.ps1 -RepoName $RepoName -Name $Name -ExtraArgs $extraArgs exit $LASTEXITCODE diff --git a/ci/select-options.Tests.ps1 b/ci/select-options.Tests.ps1 new file mode 100644 index 000000000..e535a78a4 --- /dev/null +++ b/ci/select-options.Tests.ps1 @@ -0,0 +1,67 @@ +# Pester 5 tests of select-options.ps1, which chooses the options.json +# entries a pull request is built on from the files it changes. + +BeforeAll { + $Script = Join-Path $PSScriptRoot 'select-options.ps1' + $Options = Get-Content -Raw (Join-Path $PSScriptRoot 'options.json') + + $Package = 'pipeline/pipeline.core/src/main/java/fiftyone/A.java' + $NativeFile = 'device-detection/device-detection.hash.engine.on-premise/' + + 'src/main/java/fiftyone/A.java' + + # The entries chosen for the changed files, in options.json order. + function Select-Entries( + [string[]]$Changed, + [string[]]$CommentOnly = @()) { + $entries = @(ConvertFrom-Json $Options -AsHashtable) + @(& $Script -Options $entries -ChangedFiles $Changed ` + -CommentOnlyFiles $CommentOnly 6>$null) + } + + function Select-Names([string[]]$Changed, [string[]]$CommentOnly = @()) { + (Select-Entries $Changed $CommentOnly | ForEach-Object { $_.Name }) -join ',' + } +} + +Describe 'select-options.ps1' { + It 'builds the standard set for a change to a package' { + Select-Names $Package | Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8' + } + + It 'adds Windows and Linux ARM for a change to a native module' { + Select-Names $NativeFile | + Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8,Ubuntu_ARM_Java_17,Windows_Java_17' + } + + It 'leaves a native module change to its comments to the standard set' { + Select-Names $NativeFile $NativeFile | Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8' + } + + It 'adds Windows and Linux ARM at depth deep for the cxx submodule' { + $entries = Select-Entries 'cxx' + ($entries | ForEach-Object { $_.Name }) -join ',' | + Should -Be 'Ubuntu_Java_17,Ubuntu_Java_8,Ubuntu_ARM_Java_17,Windows_Java_17' + $entries | ForEach-Object { $_.Depth | Should -Be 'deep' } + } + + It 'adds JDK 11 and 21 for a pom.xml change' { + Select-Names 'pipeline/pipeline.core/pom.xml' | + Should -Be 'Ubuntu_Java_21,Ubuntu_Java_17,Ubuntu_Java_11,Ubuntu_Java_8' + } + + It 'builds every entry but macOS, at depth deep, for a change outside the package trees' { + $entries = Select-Entries 'ci/run-unit-tests.ps1' + ($entries | ForEach-Object { $_.Name }) -join ',' | Should -Be ( + 'Ubuntu_Java_21,Ubuntu_Java_17,Ubuntu_Java_11,Ubuntu_Java_8,' + + 'Ubuntu_ARM_Java_17,Windows_Java_21,Windows_Java_17,' + + 'Windows_Java_11,Windows_Java_8') + $entries | ForEach-Object { $_.Depth | Should -Be 'deep' } + } + + It 'runs the Selenium contract on Ubuntu JDK 17 alone, and no performance tests' { + $entries = Select-Entries 'ci/run-unit-tests.ps1' + ($entries | Where-Object { $_.SeleniumContract } | + ForEach-Object { $_.Name }) -join ',' | Should -Be 'Ubuntu_Java_17' + $entries | Where-Object { $_.RunPerformance } | Should -BeNullOrEmpty + } +} diff --git a/ci/select-options.ps1 b/ci/select-options.ps1 new file mode 100644 index 000000000..1fb18ac53 --- /dev/null +++ b/ci/select-options.ps1 @@ -0,0 +1,70 @@ +param( + # Every entry of options.json. + [Parameter(Mandatory)][object[]]$Options, + # The files the pull request changes, less documentation. + [string[]]$ChangedFiles, + # Those of the changed files whose change touches only comments. + [string[]]$CommentOnlyFiles +) +# Chooses the options.json entries a pull request is built on, for common-ci's +# steps/select-options.ps1. The branch itself and a release build every entry. +# +# The standard set is JDK 17 and JDK 8 on Linux, the main JDK and the oldest, +# without the three IP Intelligence tests that load the Enterprise data file, +# and with the Selenium contract on JDK 17 alone. A change in more than its +# comments to a native module or the cxx submodule adds Windows and Linux ARM, +# where the native library is built differently. A change to IP Intelligence +# or the cxx submodule runs the three tests, and a change to a pom.xml adds +# JDK 11 and 21. A change to a file outside the package trees, such as ci, +# counts for every entry. Only five macOS jobs run at once across the +# organisation, and the releases need them, so macOS is left to the branch and +# release builds, as is performance. + +$Standard = @('Ubuntu_Java_17', 'Ubuntu_Java_8') +$Contract = 'Ubuntu_Java_17' +$NativeEntries = @('Windows_Java_17', 'Ubuntu_ARM_Java_17') +$Jdk = @('Ubuntu_Java_11', 'Ubuntu_Java_21') + +$PackageTrees = @('pipeline/', 'device-detection/', 'ip-intelligence/', + 'location/', 'examples/') +$NativePaths = @('device-detection/device-detection.hash.engine.on-premise/', + 'ip-intelligence/ip-intelligence.engine.on-premise/') +$IpIntelligencePaths = @('ip-intelligence/', 'examples/ip-intelligence/') + +function Test-Under([string]$Path, [string[]]$Prefixes) { + return @($Prefixes | Where-Object { $Path.StartsWith($_) }).Count -gt 0 +} + +$code = @($ChangedFiles | Where-Object { $CommentOnlyFiles -notcontains $_ }) +$cxx = @($code | Where-Object { $_ -eq 'cxx' }).Count -gt 0 +$native = $cxx -or @($code | Where-Object { Test-Under $_ $NativePaths }).Count -gt 0 +$ipi = $cxx -or + @($code | Where-Object { Test-Under $_ $IpIntelligencePaths }).Count -gt 0 +$poms = @($code | Where-Object { $_ -match '(^|/)pom\.xml$' }).Count -gt 0 +$other = @($code | Where-Object { + $_ -ne 'cxx' -and $_ -notmatch '(^|/)pom\.xml$' -and + !(Test-Under $_ $PackageTrees) +}).Count -gt 0 +Write-Host ("Native: $native. IP Intelligence: $ipi. pom.xml: $poms. " + + "Outside the package trees: $other.") + +foreach ($entry in $Options) { + $name = $entry.Name + if ($entry.Image -like 'macos*') { + Write-Host "$name is left to the branch and release builds" + continue + } + $why = if ($Standard -contains $name) { 'the standard set' } + elseif ($other) { 'a change outside the package trees' } + elseif ($native -and $NativeEntries -contains $name) { 'a native change' } + elseif ($poms -and $Jdk -contains $name) { 'a pom.xml change' } + if (!$why) { + Write-Host "$name is not needed for this change" + continue + } + $entry.Depth = ($ipi -or $other) ? 'deep' : 'standard' + $entry.SeleniumContract = $name -eq $Contract + $entry.RunPerformance = $false + Write-Host "$name runs for $why, at depth $($entry.Depth)" + $entry +} diff --git a/ci/setup-environment.ps1 b/ci/setup-environment.ps1 index ac900bbd4..23e2d11a9 100644 --- a/ci/setup-environment.ps1 +++ b/ci/setup-environment.ps1 @@ -1,6 +1,8 @@ param( [Parameter(Mandatory)][string]$RepoName, - [Parameter(Mandatory)][string]$JavaSDKEnvVar + [Parameter(Mandatory)][string]$JavaSDKEnvVar, + # standard, from select-options.ps1, leaves out the tests WMIC is for. + [string]$Depth ) $ErrorActionPreference = "Stop" $PSNativeCommandUseErrorActionPreference = $true @@ -11,7 +13,9 @@ $PSNativeCommandUseErrorActionPreference = $true # Without WMIC a forked test JVM on Windows can end itself part way through # the long running data file tests. The script explains why. -& "$PSScriptRoot/install-wmic.ps1" +if ($Depth -ne 'standard') { + & "$PSScriptRoot/install-wmic.ps1" +} # The native builds run PreBuild.sh, which has to be executable whatever the # checkout did to its mode. diff --git a/ci/standard-excludes.txt b/ci/standard-excludes.txt new file mode 100644 index 000000000..03d3b4b92 --- /dev/null +++ b/ci/standard-excludes.txt @@ -0,0 +1,8 @@ +# Tests that the standard set of a pull request leaves out, for +# run-unit-tests.ps1. Each loads the multi-gigabyte IP Intelligence Enterprise +# data file. +**/fiftyone/ipintelligence/IPIntelligenceTests.* +**/fiftyone/ipintelligence/examples/console/MetadataOnPremTest.* +**/fiftyone/ipintelligence/examples/console/CompareOnPremTest.* +# Surefire's own default, which a file of excludes replaces. +**/*$*