diff --git a/.github/workflows/nightly-pipeline.yml b/.github/workflows/nightly-pipeline.yml index e44f9d29..7f434943 100644 --- a/.github/workflows/nightly-pipeline.yml +++ b/.github/workflows/nightly-pipeline.yml @@ -15,53 +15,59 @@ on: default: '' schedule: # The 51degrees-cxx nightly starts at 00:35 and ends by about 02:35, so - # its release reaches this submodule update the same night. - - cron: '0 3 * * *' + # its release reaches this submodule update the same night. The Java and + # Node nightlies start at 03:00 and want macOS runners too, so this one + # starts after them. + - cron: '0 5 * * *' jobs: - SubmoduleUpdate: - name: Nightly Submodule Update - uses: 51Degrees/common-ci/.github/workflows/nightly-submodule-update.yml@main + # The publish builds main, which the pull requests into rc do not change, so + # it runs first and has the runners before the pull request builds do. + Publish: + name: Nightly Publish + uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main with: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} + cache-assets: true secrets: token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} + CsvUrl: ${{ secrets.CSV_URL }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + PypiToken: ${{ secrets.PYPI_TOKEN }} + TestPypiToken: ${{ secrets.TEST_PYPI_TOKEN }} - PullRequests: - if: ${{ !cancelled() }} - needs: SubmoduleUpdate - name: Nightly Pull Requests - uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main + SubmoduleUpdate: + name: Nightly Submodule Update + uses: 51Degrees/common-ci/.github/workflows/nightly-submodule-update.yml@main with: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - skip-lite-tests: ${{ inputs.skip-lite-tests || false }} - cache-assets: true secrets: token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - Publish: + PullRequests: if: ${{ !cancelled() }} - needs: PullRequests - name: Nightly Publish - uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main + needs: [Publish, SubmoduleUpdate] + name: Nightly Pull Requests + uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main with: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} + skip-lite-tests: ${{ inputs.skip-lite-tests || false }} cache-assets: true + # One configuration that fails does not cancel the others. + fail-fast: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} @@ -70,5 +76,3 @@ jobs: IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - PypiToken: ${{ secrets.PYPI_TOKEN }} - TestPypiToken: ${{ secrets.TEST_PYPI_TOKEN }} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 0d600a54..10b95bd4 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -13,6 +13,10 @@ on: common-ci-ref: type: string default: '' + full-matrix: + description: 'Build every configuration, not only those the changes need' + type: boolean + default: false jobs: PullRequests: @@ -25,6 +29,12 @@ jobs: common-ci-ref: ${{ inputs.common-ci-ref || '' }} skip-lite-tests: ${{ inputs.skip-lite-tests || false }} cache-assets: true + # Each pull request is built on the configurations its changes need, + # as ci/select-options.ps1 chooses, and one that fails does not cancel + # the others. With nothing eligible, the nightly builds rc instead. + full-matrix: ${{ inputs.full-matrix || false }} + fail-fast: false + build-without-pull-requests: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} diff --git a/.github/workflows/release-pull-request.yml b/.github/workflows/release-pull-request.yml new file mode 100644 index 00000000..8af10442 --- /dev/null +++ b/.github/workflows/release-pull-request.yml @@ -0,0 +1,32 @@ +name: Release Pull Request + +# Builds and tests the release pull request, rc into main, on every entry in +# ci/options.json, and never merges it. A pull request into rc is built only on +# the entries its changes need, so this is the full build of a release before +# it reaches main. See common-ci .github/workflows/release-pull-request.yml. +on: + pull_request: + branches: [main] + +# A push to rc while the release is open builds it again, in place of the +# build of the older rc. +concurrency: + group: release-pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + Release: + name: Release Pull Request + uses: 51Degrees/common-ci/.github/workflows/release-pull-request.yml@main + with: + repo-name: ${{ github.event.repository.name }} + org-name: ${{ github.event.repository.owner.login }} + cache-assets: true + secrets: + token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} + CsvUrl: ${{ secrets.CSV_URL }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} diff --git a/AGENTS.md b/AGENTS.md index 39b710d7..6d699465 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -262,7 +262,16 @@ product is built or tested belongs in that product's `ci/`, not in the root. A change touches a product when it is in that product's folder, or in a package another product depends on; a change outside every product folder touches all of them ([ci/packages.ps1](ci/packages.ps1), -`Resolve-AffectedProducts`). +`Resolve-AffectedProducts`). A pull request's changes are measured from the +branch it merges into, `rc`. + +[ci/select-options.ps1](ci/select-options.ps1) chooses the entries a pull +request runs. The standard set is Python 3.14 and 3.10 on Linux, with the +Selenium contract on 3.14 alone. A native change adds Windows, a packaging +change adds the other Python versions on Linux, and a change outside the +product folders or to a `ci/` folder adds every entry but macOS. macOS and +performance run only when `rc` itself or the release pull request is built, +which `.github/workflows/release-pull-request.yml` does on every entry. ## Versions are per package diff --git a/ci/build-project.ps1 b/ci/build-project.ps1 index 3461d091..639d7435 100644 --- a/ci/build-project.ps1 +++ b/ci/build-project.ps1 @@ -1,5 +1,8 @@ param ( - [Parameter(Mandatory)][string]$RepoName + [Parameter(Mandatory)][string]$RepoName, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/dispatch.ps1 b/ci/dispatch.ps1 index 4249d40b..06ba77ef 100644 --- a/ci/dispatch.ps1 +++ b/ci/dispatch.ps1 @@ -10,12 +10,15 @@ # Runs $Step for each affected product, or for each of $Products when given. # Every product is run even when an earlier one fails, so one run reports -# every broken product, and the step fails at the end if any did. +# every broken product, and the step fails at the end if any did. A pull +# request's changes are measured from the Branch in $Options, the branch it +# merges into. function Invoke-ProductStep([string]$Step, [string]$RepoName, [Collections.IDictionary]$Options, [string[]]$Products) { $failed = [Collections.Generic.List[string]]::new() if (-not $Products) { - $Products = Get-AffectedProducts $RepoName + $branch = $Options.Branch ? $Options.Branch : 'main' + $Products = Get-AffectedProducts $RepoName $branch } foreach ($product in $Products) { $script = Join-Path $RepoName $product 'ci' "$Step.ps1" diff --git a/ci/fetch-assets.ps1 b/ci/fetch-assets.ps1 index 802476f1..7a37794a 100644 --- a/ci/fetch-assets.ps1 +++ b/ci/fetch-assets.ps1 @@ -8,7 +8,10 @@ param ( [string]$DeviceDetectionUrl, [string]$CsvUrl, [string]$IpIntelligence, - [string]$IpIntelligenceUrl + [string]$IpIntelligenceUrl, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 429c43ff..d9a924bf 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -168,15 +168,18 @@ function Resolve-AffectedProducts($Packages, [string[]]$Changed) { } # The products a pull request changes, or every product when the checkout is -# not a pull request (a publish, or a run on main), as 51degrees-cxx does. -function Get-AffectedProducts([string]$RepoPath) { - # merge-base fails where there is no origin/main, which is an answer here. +# not a pull request (a publish, or a build of the branch itself), as +# 51degrees-cxx does. A pull request is checked out with the branch it merges +# into merged in, so the merge base with that branch leaves only the pull +# request's own changes. +function Get-AffectedProducts([string]$RepoPath, [string]$Branch = 'main') { + # merge-base fails where there is no such branch, which is an answer here. # Set for this function only, so a caller that stops on a failing native # command, as publish-package.ps1 does, is not stopped by it. $PSNativeCommandUseErrorActionPreference = $false $packages = Get-Packages $RepoPath $head = git -C $RepoPath rev-parse HEAD - $base = git -C $RepoPath merge-base origin/main HEAD 2>$null + $base = git -C $RepoPath merge-base "origin/$Branch" HEAD 2>$null if ($LASTEXITCODE -ne 0 -or $null -eq $base -or $base -eq $head) { $global:LASTEXITCODE = 0 return @($packages.Product | Select-Object -Unique) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index ae1c20f8..59885cae 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -1,6 +1,12 @@ param ( [Parameter(Mandatory)][string]$RepoName, - [Parameter(Mandatory)][hashtable]$Keys + [Parameter(Mandatory)][hashtable]$Keys, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main', + # False, from select-options.ps1, on every entry of a pull request but + # one, so the browser contract runs once. + [bool]$SeleniumContract = $true ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/run-performance-tests.ps1 b/ci/run-performance-tests.ps1 index 58401f22..4fa56758 100644 --- a/ci/run-performance-tests.ps1 +++ b/ci/run-performance-tests.ps1 @@ -1,7 +1,10 @@ param ( [Parameter(Mandatory)][string]$RepoName, [string]$Name, - [bool]$RunPerformance + [bool]$RunPerformance, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index e74a60ff..38e3ee6e 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -1,5 +1,8 @@ param ( - [Parameter(Mandatory)][string]$RepoName + [Parameter(Mandatory)][string]$RepoName, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/select-options.Tests.ps1 b/ci/select-options.Tests.ps1 new file mode 100644 index 00000000..9e0042dc --- /dev/null +++ b/ci/select-options.Tests.ps1 @@ -0,0 +1,69 @@ +# Tests for ci/select-options.ps1, with the entries of the real options.json +# read as common-ci's steps/select-options.ps1 reads them, as hashtables. + +BeforeAll { + $script:Select = Join-Path $PSScriptRoot 'select-options.ps1' + + # The names of the entries chosen for the changed files given, and the + # entries themselves in $script:Chosen. + function Get-Chosen([string[]]$ChangedFiles, [string[]]$CommentOnlyFiles = @()) { + $options = @(Get-Content -Raw (Join-Path $PSScriptRoot 'options.json') | + ConvertFrom-Json -AsHashtable) + $script:Chosen = @(& $script:Select -Options $options ` + -ChangedFiles $ChangedFiles -CommentOnlyFiles $CommentOnlyFiles 6>$null) + return @($script:Chosen | ForEach-Object { $_.Name } | Sort-Object) + } + + $script:StandardSet = @('Ubuntu_Python_3.10', 'Ubuntu_Python_3.14') +} + +Describe 'select-options.ps1' { + It 'builds the standard set for a change to a package''s code' { + Get-Chosen @('pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py') | + Should -Be $script:StandardSet + } + + It 'runs the Selenium contract on Python 3.14 alone, and no performance tests' { + $null = Get-Chosen @('location/src/fiftyone_location/location_cloud.py') + foreach ($entry in $script:Chosen) { + $entry.SeleniumContract | Should -Be ($entry.Name -eq 'Ubuntu_Python_3.14') + $entry.RunPerformance | Should -BeFalse + } + } + + It 'adds Windows for a change to an on-premise package' { + Get-Chosen @('device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py') | + Should -Be (@($script:StandardSet + 'Windows_Python_3.14') | Sort-Object) + } + + It 'adds Windows for a change to the cxx submodule' { + Get-Chosen @('cxx') | + Should -Be (@($script:StandardSet + 'Windows_Python_3.14') | Sort-Object) + } + + It 'leaves Windows out for a change only to the comments of an on-premise file' { + $file = 'ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py' + Get-Chosen @($file) @($file) | Should -Be $script:StandardSet + } + + It 'adds Python 3.11 to 3.13 on Linux for a change to a setup.py' { + Get-Chosen @('location/setup.py') | Should -Be (@($script:StandardSet + + 'Ubuntu_Python_3.11' + 'Ubuntu_Python_3.12' + 'Ubuntu_Python_3.13') | + Sort-Object) + } + + It 'builds every entry but macOS for a change to ' -ForEach @( + @{ Case = 'a ci folder'; File = 'device-detection/ci/build-project.ps1' } + @{ Case = 'the root ci folder'; File = 'ci/packages.ps1' } + @{ Case = 'a file outside the product folders'; File = '.github/workflows/pull-requests.yml' } + ) { + $chosen = Get-Chosen @($File) + $chosen.Count | Should -Be 10 + $chosen | Where-Object { $_ -like 'macOS*' } | Should -BeNullOrEmpty + } + + It 'never builds macOS on a pull request' { + Get-Chosen @('cxx', 'ci/packages.ps1', 'location/setup.py') | + Where-Object { $_ -like 'macOS*' } | Should -BeNullOrEmpty + } +} diff --git a/ci/select-options.ps1 b/ci/select-options.ps1 new file mode 100644 index 00000000..cc22871f --- /dev/null +++ b/ci/select-options.ps1 @@ -0,0 +1,73 @@ +param( + # Every entry of options.json. + [Parameter(Mandatory)][object[]]$Options, + # The files the pull request changes, less documentation. + [string[]]$ChangedFiles, + # Those of the changed files whose change touches only comments. + [string[]]$CommentOnlyFiles +) +# Chooses the options.json entries a pull request is built on, for common-ci's +# steps/select-options.ps1. The branch itself and a release build every entry. +# Within each entry, packages.ps1 still builds only the products the change +# touches. +# +# The standard set is Python 3.14 and 3.10 on Linux, the newest and the oldest, +# with the Selenium contract on 3.14 alone. A change in more than its comments +# to an on-premise package, its data or the cxx submodule adds Windows, where +# the native module is built differently. A change to a setup.py, setup.cfg or +# pyproject.toml adds Python 3.11 to 3.13 on Linux, and a change outside the +# product folders or to a ci folder adds every entry. Only five macOS jobs run +# at once across the organisation, and the releases need them, so macOS is left +# to the branch and release builds, as is performance. + +$Standard = @('Ubuntu_Python_3.14', 'Ubuntu_Python_3.10') +$Contract = 'Ubuntu_Python_3.14' +$Native = @('Windows_Python_3.14') +$Versions = @('Ubuntu_Python_3.13', 'Ubuntu_Python_3.12', 'Ubuntu_Python_3.11') +$ProductFolders = @('pipeline/', 'device-detection/', 'ip-intelligence/', + 'location/') +# The data files come in through the cxx submodule, so a change to them is a +# change to cxx. +$NativePaths = @('device-detection/fiftyone_devicedetection_onpremise/', + 'ip-intelligence/fiftyone_ipintelligence_onpremise/') + +function Test-Under([string]$Path, [string[]]$Prefixes) { + return @($Prefixes | Where-Object { + $Path -eq $_.TrimEnd('/') -or $Path.StartsWith($_.TrimEnd('/') + '/') + }).Count -gt 0 +} + +# PowerShell names are not case sensitive, so these must not share a name with +# the lists above, such as $Native. +$code = @($ChangedFiles | Where-Object { $CommentOnlyFiles -notcontains $_ }) +$nativeChanged = @($code | Where-Object { + $_ -eq 'cxx' -or (Test-Under $_ $NativePaths) +}).Count -gt 0 +$setupChanged = @($code | Where-Object { + $_ -match '(^|/)(setup\.py|setup\.cfg|pyproject\.toml)$' +}).Count -gt 0 +$otherChanged = @($code | Where-Object { + $_ -ne 'cxx' -and (!(Test-Under $_ $ProductFolders) -or $_ -match '(^|/)ci/') +}).Count -gt 0 +Write-Host ("Native: $nativeChanged. Setup: $setupChanged. " + + "Outside the product folders or in a ci folder: $otherChanged.") + +foreach ($entry in $Options) { + $name = $entry.Name + if ($entry.Image -like 'macos*') { + Write-Host "$name is left to the branch and release builds" + continue + } + $why = if ($Standard -contains $name) { 'the standard set' } + elseif ($otherChanged) { 'a change outside the product folders or to CI' } + elseif ($nativeChanged -and $Native -contains $name) { 'a native change' } + elseif ($setupChanged -and $Versions -contains $name) { 'a packaging change' } + if (!$why) { + Write-Host "$name is not needed for this change" + continue + } + $entry.SeleniumContract = $name -eq $Contract + $entry.RunPerformance = $false + Write-Host "$name runs for $why" + $entry +} diff --git a/ci/setup-environment.ps1 b/ci/setup-environment.ps1 index 932f5d81..24c1b8bd 100644 --- a/ci/setup-environment.ps1 +++ b/ci/setup-environment.ps1 @@ -1,6 +1,9 @@ param ( [Parameter(Mandatory)][string]$RepoName, - [Parameter(Mandatory)][string]$LanguageVersion + [Parameter(Mandatory)][string]$LanguageVersion, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index 9c45928e..ed7a17d2 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -2,7 +2,10 @@ param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] - [Hashtable]$Keys + [Hashtable]$Keys, + # False, from the root select-options.ps1, on every entry of a pull + # request but one, so the browser contract runs once. + [bool]$SeleniumContract = $true ) if ($IsWindows) { @@ -47,6 +50,14 @@ if ($LASTEXITCODE -ne 0) { $failures += "the Python integration tests exited with code $LASTEXITCODE" } +if (-not $SeleniumContract) { + Write-Output "Another entry runs the Selenium contract tests for this pull request." + if ($failures.Count -gt 0) { + throw "Integration tests failed: $($failures -join '; ')" + } + return +} + # The shared Selenium contract tests are run against the cloud example and the # on-premise example in turn. Each result is also written as a trx file next # to the other integration results, so it is published with them.