From 15978386c1241ecc6945224d6e606257fd871030 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 5 Nov 2026 21:09:25 +0000 Subject: [PATCH 01/13] Use common-ci's release plan in place of this repository's copy The functions that work out which packages a publish releases, and at which version, move to common-ci as steps/release-plan.ps1, so that every package repository can use the same ones. ci/packages.ps1 now dot-sources that step and keeps only what is particular to this repository: the tox.ini check and the products a pull request touches. The moved functions are the same code, so every script that dot-sources ci/packages.ps1 calls them as before. common-ci runs these scripts from its own root, with this repository checked out inside it, which is where ci/packages.ps1 looks first. It also finds a checkout of common-ci beside this repository, as ci/dispatch.Tests.ps1 does, or one named by COMMON_CI. The release plan's tests move to common-ci with it, and ci/packages.Tests.ps1 keeps the tests of what stays here. --- ci/packages.Tests.ps1 | 179 +----------------------------------- ci/packages.ps1 | 207 ++++-------------------------------------- 2 files changed, 23 insertions(+), 363 deletions(-) diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 2a2887d9..10b2fb21 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -1,9 +1,9 @@ # Tests for ci/packages.ps1. # # The selection rules are tested against a small package list written here, so -# a test states exactly which graph it relies on. The release plan reads tags -# and diffs, so those tests build a throwaway git repository. One test reads the -# real packages.json, to catch a typo in a dependency name or a cycle. +# a test states exactly which graph it relies on. Some tests read the real +# packages.json, to catch a typo in a dependency name or a cycle. The release +# plan, which packages.ps1 loads from common-ci, is tested there. BeforeAll { . (Join-Path $PSScriptRoot 'packages.ps1') @@ -34,81 +34,10 @@ BeforeAll { New-Package 'ipi' 'ipi' 'ipi/ipi' @('engines') -FirstVersion '4.5.0' New-Package 'examples' 'device' 'device/examples' @('dd') -Publish $false ) - - # A repository with the given files committed and the given tags on that - # commit. Further commits are made with Add-Commit. - function New-Repo([string[]]$Files, [string[]]$Tags) { - $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - git init --quiet $path - Add-Commit $path $Files - foreach ($tag in $Tags) { - git -C $path tag $tag - } - return $path - } - - function Add-Commit([string]$Repo, [string[]]$Files) { - foreach ($file in $Files) { - $full = Join-Path $Repo $file - $null = New-Item -ItemType Directory -Force (Split-Path $full) - Add-Content -Path $full -Value ([Guid]::NewGuid()) - } - git -C $Repo add -A - git -C $Repo -c user.name=test -c user.email=test@example.com ` - commit --quiet --allow-empty -m 'Change' - } - - $script:AllFiles = @( - 'pipeline/core/a.py', 'pipeline/engines/a.py', 'pipeline/did/a.py', - 'pipeline/owid/a.py', 'device/dd/a.py', 'device/examples/a.py', - 'location/setup.py', 'ipi/ipi/a.py') - $script:AllTags = @('core/4.5.38', 'engines/4.5.38', 'did/4.5.38', - 'dd/4.5.96', 'location/4.5.4') } Describe 'packages.ps1' { -AfterEach { - if ($script:Repo -and (Test-Path $script:Repo)) { - Remove-Item -Recurse -Force $script:Repo - } - $script:Repo = $null -} - -Describe 'Get-DependentClosure' { - It 'adds every package that depends on a changed one, through others' { - $closure = Get-DependentClosure $script:Graph @('core') - $closure | Sort-Object | Should -Be @( - 'core', 'dd', 'engines', 'examples', 'ipi', 'location') - } - - It 'adds nothing for a package nothing depends on' { - Get-DependentClosure $script:Graph @('did') | Should -Be @('did') - } - - It 'is empty when given nothing' { - @(Get-DependentClosure $script:Graph @()).Count | Should -Be 0 - } -} - -Describe 'Sort-ByDependency' { - It 'puts every package after the packages it depends on' { - $order = Sort-ByDependency $script:Graph @('dd', 'engines', 'core') - $order | Should -Be @('core', 'engines', 'dd') - } - - It 'ignores dependencies that are not being sorted' { - Sort-ByDependency $script:Graph @('dd') | Should -Be @('dd') - } - - It 'refuses a cycle rather than looping' { - $cycle = @( - New-Package 'a' 'p' 'p/a' @('b') - New-Package 'b' 'p' 'p/b' @('a')) - { Sort-ByDependency $cycle @('a', 'b') } | Should -Throw '*cycle*' - } -} - Describe 'Resolve-AffectedProducts' { It 'tests only the product of a package nothing depends on' { Resolve-AffectedProducts $script:Graph @('device/dd/a.py') | @@ -147,108 +76,6 @@ Describe 'Resolve-AffectedProducts' { } } -Describe 'Get-ReleasePlan' { - It 'releases nothing but first releases when nothing changed since the tags' { - $script:Repo = New-Repo $script:AllFiles $script:AllTags - Add-Commit $script:Repo @() - $plan = @(Get-ReleasePlan $script:Repo $script:Graph) - $plan.Name | Should -Be @('ipi') - $plan[0].Version | Should -Be '4.5.0' - $plan[0].Reason | Should -Be 'first release' - } - - It 'releases a changed package at the next patch, and its dependents' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('pipeline/engines/a.py') - $plan = @(Get-ReleasePlan $script:Repo $script:Graph) - $plan.Name | Should -Be @('engines', 'dd', 'ipi') - ($plan | Where-Object Name -eq 'engines').Version | Should -Be '4.5.39' - ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.97' - ($plan | Where-Object Name -eq 'dd').Reason | - Should -Be 'a package it depends on is released' - ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.1' - } - - It 'releases a vendoring package when the vendored folder changes' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('pipeline/owid/a.py') - @(Get-ReleasePlan $script:Repo $script:Graph).Name | Should -Be @('did') - } - - It 'releases nothing for a change to an unpublished package' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/examples/a.py') - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'releases nothing for a change outside every package' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('ci/build.ps1', 'device/docs/a.md') - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'releases nothing a second time from a commit already released' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/a.py') - git -C $script:Repo tag 'dd/4.5.97' - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'continues from the highest tag, not the latest one made' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + @( - 'ipi/4.5.0', 'dd/4.5.9')) - Add-Commit $script:Repo @('device/dd/a.py') - (@(Get-ReleasePlan $script:Repo $script:Graph) | - Where-Object Name -eq 'dd').Version | Should -Be '4.5.97' - } - - It 'does not read another package''s tags as its own' { - # 'ipi' must not see tags of a package whose name starts with 'ipi'. - $graph = $script:Graph + (New-Package 'ipi_cloud' 'ipi' 'ipi/cloud' -FirstVersion '4.5.0') - $script:Repo = New-Repo ($script:AllFiles + 'ipi/cloud/a.py') ( - $script:AllTags + 'ipi_cloud/4.5.7') - $plan = @(Get-ReleasePlan $script:Repo $graph) - ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.0' - } - - It 'plans a changed package for a caller that stops on native errors' { - # publish-package.ps1 stops on a failing native command; 'git diff - # --quiet' exiting 1 for a changed package once failed the publish. - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/a.py') - $plan = & { - $ErrorActionPreference = 'Stop' - $PSNativeCommandUseErrorActionPreference = $true - @(Get-ReleasePlan $script:Repo $script:Graph) - } - $plan.Name | Should -Be @('dd') - } - - It 'reads the tags whatever folder it runs from' { - # PowerShell expands a wildcard in a native command's argument into - # matching files, so a tag pattern run from a folder that holds a - # folder of the same name, as the repository root holds 'location', - # once found no tags. - $script:Repo = New-Repo $script:AllFiles $script:AllTags - $here = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - $null = New-Item -ItemType Directory -Force (Join-Path $here 'location') - $null = New-Item -ItemType File (Join-Path $here 'location' 'setup.py') - Push-Location $here - try { - (Get-ReleasedVersions $script:Repo 'location')[-1] | Should -Be ([version]'4.5.4') - } finally { - Pop-Location - Remove-Item -Recurse -Force $here - } - } - - It 'refuses a never-released package with no FirstVersion' { - $graph = @(New-Package 'new' 'p' 'p/new') - $script:Repo = New-Repo @('p/new/a.py') @() - { Get-ReleasePlan $script:Repo $graph } | Should -Throw '*FirstVersion*' - } -} - Describe 'the real packages.json' { BeforeAll { $script:Real = Get-Packages (Split-Path $PSScriptRoot) diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 70e6af0c..1456101b 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -1,34 +1,25 @@ -# Shared by the scripts in this folder, which dot-source it. Reads the -# packages this repository publishes from packages.json, works out which -# products a pull request touches, and which packages a publish releases at -# which version. +# Shared by the scripts in this folder, which dot-source it. Works out which +# products a pull request touches, and checks each package's tox.ini. # -# Each package has its own version, tagged '/'. A package is -# released when a file under its folder changed since its last tag, or when a -# package it depends on is being released, because its requirement on that -# package moves. This is the scheme in 51Degrees/rust#205, which the per -# language repositories follow. - -# Reads packages.json. Kept as a function so the tests can give their own list. -function Get-Packages([string]$RepoPath) { - return Get-Content -Raw (Join-Path $RepoPath 'ci/packages.json') | - ConvertFrom-Json -} - -# The folders whose changes count as a change to the package: its own folder, -# and any folder it takes files from at build time, such as a submodule it -# vendors. -function Get-PackagePaths($Package) { - $paths = @($Package.Path) - if ($Package.ExtraPaths) { - $paths += $Package.ExtraPaths - } - return $paths -} - -function Test-PathUnder([string]$Path, [string]$Folder) { - return $Path -eq $Folder -or $Path.StartsWith("$Folder/") +# Which packages a publish releases, and at which version, is worked out by +# common-ci's steps/release-plan.ps1 from ci/packages.json. common-ci runs +# these scripts from its own root, with this repository checked out inside +# it. A checkout of common-ci beside this repository is found too, or one +# elsewhere can be named with COMMON_CI. +$releasePlan = @( + $env:COMMON_CI, + (Join-Path $PSScriptRoot '..' '..'), + (Join-Path $PSScriptRoot '..' '..' 'common-ci')) | + Where-Object { $_ } | + ForEach-Object { Join-Path $_ 'steps' 'release-plan.ps1' } | + Where-Object { Test-Path $_ } | + Select-Object -First 1 +if ($null -eq $releasePlan) { + throw "common-ci's steps/release-plan.ps1 was not found. Set COMMON_CI " + + "to a checkout of common-ci." } +. $releasePlan +Remove-Variable releasePlan # Each tox.ini that does not install every package its package depends on, # directly or through another, from this checkout, as a line naming what is @@ -75,56 +66,6 @@ function Get-ToxGaps([string]$RepoPath, $Packages = $null) { return $gaps } -# The names of the given packages and of every package that depends on one of -# them, directly or through another. -function Get-DependentClosure($Packages, [string[]]$Names) { - $closure = [Collections.Generic.HashSet[string]]::new() - foreach ($name in @($Names | Where-Object { $_ })) { - $null = $closure.Add($name) - } - $grew = $true - while ($grew -eq $true) { - $grew = $false - foreach ($package in $Packages) { - if ($closure.Contains($package.Name) -eq $true) { - continue - } - foreach ($dependency in $package.DependsOn) { - if ($closure.Contains($dependency) -eq $true) { - $null = $closure.Add($package.Name) - $grew = $true - break - } - } - } - } - return @($closure) -} - -# The named packages, each after every package it depends on, so a release -# never publishes a package before the version it requires exists. -function Sort-ByDependency($Packages, [string[]]$Names) { - $remaining = [Collections.Generic.List[string]]::new([string[]]@($Names)) - $sorted = [Collections.Generic.List[string]]::new() - while ($remaining.Count -gt 0) { - $ready = @($remaining | Where-Object { - $name = $_ - $package = $Packages | Where-Object { $_.Name -eq $name } - @($package.DependsOn | Where-Object { - $remaining.Contains($_) -eq $true - }).Count -eq 0 - } | Sort-Object) - if ($ready.Count -eq 0) { - throw "packages.json has a dependency cycle among: $($remaining -join ', ')" - } - foreach ($name in $ready) { - $sorted.Add($name) - $null = $remaining.Remove($name) - } - } - return @($sorted) -} - # The products a set of changed files touches. A file in a package's folders # touches that package's product and the product of every package depending on # it, because a change to pipeline can break device-detection. @@ -184,111 +125,3 @@ function Get-AffectedProducts([string]$RepoPath) { $changed = @(git -C $RepoPath diff --name-only $base $head) return Resolve-AffectedProducts $packages $changed } - -# The parts of a version such as 4.5.38, or $null for anything else. -function ConvertTo-VersionParts([string]$Text) { - $match = [regex]::Match($Text, '^(\d+)\.(\d+)\.(\d+)$') - if ($match.Success -eq $false) { - return $null - } - return [version]::new([int]$match.Groups[1].Value, - [int]$match.Groups[2].Value, [int]$match.Groups[3].Value) -} - -# The package's released versions, read from its '/' tags, -# highest last. The '/' after the name keeps fiftyone_ipintelligence from -# reading the tags of fiftyone_ipintelligence_cloud. -function Get-ReleasedVersions([string]$RepoPath, [string]$Name, - [switch]$AtHead) { - # Tags are found by prefix, not with a 'name/*' pattern: PowerShell on - # Linux and macOS expands a '*' in an argument to a native command into - # matching files, so from a folder holding a 'location' folder git would - # be handed file names and find no tags. - $arguments = @('-C', $RepoPath, 'for-each-ref', - '--format=%(refname:strip=2)', "refs/tags/$Name/") - if ($AtHead -eq $true) { - $arguments += @('--points-at', 'HEAD') - } - $versions = foreach ($tag in (& git @arguments)) { - ConvertTo-VersionParts $tag.Substring($Name.Length + 1) - } - return @($versions | Where-Object { $null -ne $_ } | Sort-Object) -} - -# What a publish of HEAD releases: one entry per package, in the order to -# publish them, each with the version it is released at and why. -# -# A package already tagged at HEAD was released from this commit, so it is -# left out, which makes a second publish of the same commit release nothing. -# A package with no tag has never been released and starts at its FirstVersion -# from packages.json. Packages that are not published, the examples, are never -# in the plan; a change to them is tested but releases nothing. -function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { - # 'git diff --quiet' exits 1 to say a package changed. That is an answer, - # not a failure, but a caller that stops on a failing native command, as - # publish-package.ps1 does, stopped there and failed the publish. Set for - # this function only. - $PSNativeCommandUseErrorActionPreference = $false - if ($null -eq $Packages) { - $Packages = Get-Packages $RepoPath - } - $published = @($Packages | Where-Object { $_.Publish -eq $true }) - - $previous = @{} - $reasons = @{} - foreach ($package in $published) { - $atHead = Get-ReleasedVersions $RepoPath $package.Name -AtHead - if ($atHead.Count -gt 0) { - continue - } - $released = Get-ReleasedVersions $RepoPath $package.Name - if ($released.Count -eq 0) { - if ([string]::IsNullOrEmpty($package.FirstVersion) -eq $true) { - throw "$($package.Name) has never been released and " + - "packages.json gives it no FirstVersion." - } - $previous[$package.Name] = $null - $reasons[$package.Name] = 'first release' - continue - } - $last = $released[-1] - $previous[$package.Name] = $last - $tag = "$($package.Name)/$last" - & git -C $RepoPath diff --quiet $tag HEAD -- (Get-PackagePaths $package) - if ($LASTEXITCODE -eq 1) { - $reasons[$package.Name] = "changed since $last" - } - $global:LASTEXITCODE = 0 - } - - # A dependent is released with its dependency, as long as it was not - # already released from this commit. - $candidates = @($published.Name | Where-Object { - $previous.ContainsKey($_) -eq $true - }) - $changed = @($reasons.Keys) - $toRelease = @(Get-DependentClosure $published $changed | Where-Object { - $candidates -contains $_ - }) - - foreach ($name in (Sort-ByDependency $published $toRelease)) { - $package = $published | Where-Object { $_.Name -eq $name } - $last = $previous[$name] - if ($null -eq $last) { - $version = $package.FirstVersion - } else { - $version = "$($last.Major).$($last.Minor).$($last.Build + 1)" - } - $reason = $reasons[$name] - if ($null -eq $reason) { - $reason = 'a package it depends on is released' - } - [pscustomobject]@{ - Name = $name - Path = $package.Path - Version = $version - Previous = $last - Reason = $reason - } - } -} From db78491866a3265370880a426e9352426d12d9a5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 17:22:08 +0000 Subject: [PATCH 02/13] Release a package only for a change that counts, leaving out documentation, tests, examples and manifest metadata --- AGENTS.md | 12 +- ci/packages.Tests.ps1 | 218 +++++++++++++++++++++++++++++++++++++ ci/packages.ps1 | 93 ++++++++++++++-- ci/release-plan-filter.ps1 | 38 +++++++ ci/release-plan-filter.py | 191 ++++++++++++++++++++++++++++++++ 5 files changed, 537 insertions(+), 15 deletions(-) create mode 100644 ci/release-plan-filter.ps1 create mode 100644 ci/release-plan-filter.py diff --git a/AGENTS.md b/AGENTS.md index 81f3a40b..08309e01 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -268,9 +268,15 @@ touches all of them ([ci/packages.ps1](ci/packages.ps1), ## Versions are per package Every package is versioned and tagged on its own, `/`. A -package is released when its folders changed since its last tag, or when a -package it depends on is released. The examples packages are tested but never -published. common-ci's publish carries one version; here it is only a label, +package is released when a file that counts in its folders changed since its +last tag, or when a package it depends on is released. Documentation +(`*.md`, `docs/`), tests (`tests/`, `test/`), examples (`examples/`) and +`LICENSE` files do not count, nor do files a package's `Ignore` globs in +`ci/packages.json` match, unless it sets `IgnoreDefaults` to false. A +`setup.py`, `setup.cfg` or `pyproject.toml` whose change is only to the +description, keywords, urls, classifiers, readme or authors does not count +either, which `ci/release-plan-filter.ps1` works out with Python. The +examples packages are tested but never published. common-ci's publish carries one version; here it is only a label, and the release plan in `ci/packages.ps1` decides what is built, published and tagged. Never push a plain `4.5.x` tag: nothing here reads one. diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 766d1380..ce3cf848 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -247,6 +247,224 @@ Describe 'Get-ReleasePlan' { $script:Repo = New-Repo @('p/new/a.py') @() { Get-ReleasePlan $script:Repo $graph } | Should -Throw '*FirstVersion*' } + + It 'says how many files counted' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/a.py', 'device/dd/b.py') + (@(Get-ReleasePlan $script:Repo $script:Graph) | + Where-Object Name -eq 'dd').Reason | + Should -Be 'changed since 4.5.96 (2 files)' + } + + It 'releases nothing for a change only to documentation, tests or examples' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @( + 'device/dd/readme.md', 'device/dd/docs/guide.txt', + 'device/dd/tests/test_a.py', 'device/dd/test/a.py', + 'device/dd/examples/a.py', 'device/dd/LICENSE', + 'pipeline/core/sub/notes.md') + @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 + } + + It 'ignores documentation, tests and examples whatever their case' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @( + 'device/dd/README.MD', 'device/dd/Tests/a.py', + 'device/dd/Examples/a.py') + @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 + } + + It 'releases for the code changed beside the documentation' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/readme.md', 'device/dd/a.py') + $plan = @(Get-ReleasePlan $script:Repo $script:Graph) + $plan.Name | Should -Be @('dd') + $plan[0].Reason | Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'leaves out the files a package''s Ignore globs match' { + $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) + ($graph | Where-Object Name -eq 'dd') | + Add-Member Ignore @('device/dd/tox.ini') + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/tox.ini') + @(Get-ReleasePlan $script:Repo $graph).Count | Should -Be 0 + } + + It 'counts documentation for a package that turns the defaults off' { + $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) + ($graph | Where-Object Name -eq 'dd') | Add-Member IgnoreDefaults $false + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/readme.md') + @(Get-ReleasePlan $script:Repo $graph).Name | Should -Be @('dd') + } +} + +Describe 'release-plan-filter.ps1' { + BeforeAll { + # A setup.py, setup.cfg and pyproject.toml like the ones here, with + # the metadata the filter leaves out and settings it does not. Line + # endings are made LF, whatever the checkout gave this file. + $script:SetupPy = @' +import setuptools + +setuptools.setup( + name="dd", + version=read("version.txt"), + author="51Degrees Engineering", + author_email="engineering@51degrees.com", + project_urls={"Source": "https://github.com/51Degrees/51degrees-python"}, + description=("Device detection."), + keywords=["51degrees", "device-detection"], + long_description=read("readme.md"), + long_description_content_type='text/markdown', + python_requires=">=3.10", + install_requires=["fiftyone_pipeline_core"], + classifiers=["Programming Language :: Python"], +) +'@.Replace("`r`n", "`n") + $script:SetupCfg = @' +[metadata] +name = dd +description = Device detection. +url = https://github.com/51Degrees/51degrees-python + +[options] +install_requires = + fiftyone_pipeline_core +'@.Replace("`r`n", "`n") + "`n" + $script:Pyproject = @' +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "dd" +description = "Device detection." +keywords = [ + "51degrees", + "device-detection", +] +dependencies = ["fiftyone_pipeline_core"] + +[project.urls] +Source = "https://github.com/51Degrees/51degrees-python" +'@.Replace("`r`n", "`n") + "`n" + + # A repository with the given files, holding the filter as this + # repository does, with the tags of the small graph and the tag of + # dd on the commit that adds the files. + function New-FilterRepo([hashtable]$Files) { + $repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + $null = New-Item -ItemType Directory -Force (Join-Path $repo 'ci') + foreach ($name in 'release-plan-filter.ps1', 'release-plan-filter.py') { + Copy-Item (Join-Path $PSScriptRoot $name) (Join-Path $repo 'ci') + } + Set-Files $repo $Files + git -C $repo tag -f 'dd/4.5.96' | Out-Null + return $repo + } + + # Commits the files with exactly the text given. + function Set-Files([string]$Repo, [hashtable]$Files) { + foreach ($file in $Files.Keys) { + $full = Join-Path $Repo $file + $null = New-Item -ItemType Directory -Force (Split-Path $full) + [IO.File]::WriteAllText($full, $Files[$file]) + } + git -C $Repo add -A + git -C $Repo -c user.name=test -c user.email=test@example.com ` + commit --quiet --allow-empty -m 'Change' + } + + function Get-DdPlan([string]$Repo) { + return @(Get-ReleasePlan $Repo $script:Graph | + Where-Object Name -eq 'dd') + } + } + + It 'does not release for a setup.py change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + $changed = $script:SetupPy + $changed = $changed.Replace('Device detection.', 'Device detection for Python.') + $changed = $changed.Replace('"device-detection"]', '"device-detection", "user-agent"]') + $changed = $changed.Replace('51degrees-python"}', '51degrees-python", "Docs": "x"}') + $changed = $changed.Replace('51Degrees Engineering', '51Degrees') + $changed = $changed.Replace('"Programming Language :: Python"]', + '"Programming Language :: Python", "Typing :: Typed"]') + $changed = $changed.Replace("'text/markdown'", "'text/x-rst'") + Set-Files $script:Repo @{ 'device/dd/setup.py' = $changed + "# A note.`n" } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a setup.py change to anything else' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + $changed = $script:SetupPy.Replace('["fiftyone_pipeline_core"]', + '["fiftyone_pipeline_core>=4.5.40"]') + Set-Files $script:Repo @{ 'device/dd/setup.py' = $changed } + $plan = Get-DdPlan $script:Repo + $plan.Count | Should -Be 1 + $plan[0].Reason | Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'counts the code changed beside a setup.py change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + Set-Files $script:Repo @{ + 'device/dd/setup.py' = $script:SetupPy.Replace( + 'Device detection.', 'Device detection for Python.') + 'device/dd/src/a.py' = 'print(1)' + } + (Get-DdPlan $script:Repo)[0].Reason | + Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'does not release for a setup.cfg change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.cfg' = $script:SetupCfg } + $changed = $script:SetupCfg.Replace('Device detection.', 'Device detection for Python.') + $changed = $changed.Replace('51degrees-python', '51degrees-python/tree/main') + Set-Files $script:Repo @{ 'device/dd/setup.cfg' = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a setup.cfg change to its options' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.cfg' = $script:SetupCfg } + Set-Files $script:Repo @{ 'device/dd/setup.cfg' = + $script:SetupCfg + " fiftyone_pipeline_engines`n" } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'does not release for a pyproject.toml change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/pyproject.toml' = $script:Pyproject } + $changed = $script:Pyproject.Replace('"Device detection."', '"Device detection for Python."') + $changed = $changed.Replace(" `"device-detection`",`n", + " `"device-detection`",`n `"user-agent`",`n") + $changed = $changed.Replace('Source = ', "Docs = `"x`"`nSource = ") + $changed += "`n[[project.authors]]`nname = `"51Degrees`"`n" + Set-Files $script:Repo @{ 'device/dd/pyproject.toml' = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a pyproject.toml change to ' -ForEach @( + @{ Was = '["fiftyone_pipeline_core"]'; Changed = '["fiftyone_pipeline_core>=4.5"]' } + @{ Was = '"setuptools>=42"'; Changed = '"setuptools>=61"' } + ) { + $script:Repo = New-FilterRepo @{ 'device/dd/pyproject.toml' = $script:Pyproject } + Set-Files $script:Repo @{ 'device/dd/pyproject.toml' = + $script:Pyproject.Replace($Was, $Changed) } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest that is new since the last tag' { + $script:Repo = New-FilterRepo @{} + Set-Files $script:Repo @{ 'device/dd/setup.py' = $script:SetupPy } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest it cannot read' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + Set-Files $script:Repo @{ 'device/dd/setup.py' = 'setup(' } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } } Describe 'Get-BuildPlan' { diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 429c43ff..ea785413 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -4,10 +4,17 @@ # which version. # # Each package has its own version, tagged '/'. A package is -# released when a file under its folder changed since its last tag, or when a -# package it depends on is being released, because its requirement on that -# package moves. This is the scheme in 51Degrees/rust#205, which the per -# language repositories follow. +# released when a file that counts under its folders changed since its last +# tag, or when a package it depends on is being released, because its +# requirement on that package moves. This is the scheme in 51Degrees/rust#205, +# which the per language repositories follow. +# +# Every file counts except those matched by the package's Ignore globs in +# packages.json and, unless its IgnoreDefaults is false, by +# $ReleasePlanDefaultIgnore, so a change only to documentation, tests or +# examples releases nothing. release-plan-filter.ps1 then leaves out a +# manifest whose change is only to metadata such as links. These are the +# fields and the filter of common-ci's steps/release-plan.ps1. # Reads packages.json. Kept as a function so the tests can give their own list. function Get-Packages([string]$RepoPath) { @@ -30,6 +37,65 @@ function Test-PathUnder([string]$Path, [string]$Folder) { return $Path -eq $Folder -or $Path.StartsWith("$Folder/") } +# The files whose changes do not release a package, being documentation, +# tests, examples and LICENSE files wherever they sit and whatever their case, +# unless the package sets IgnoreDefaults to false. These are git glob +# pathspecs from the repository root, where '**/' matches any number of +# folders, none included. +$ReleasePlanDefaultIgnore = @( + '**/*.md' + '**/docs/**' + '**/tests/**' + '**/test/**' + '**/examples/**' + '**/LICENSE*' +) + +# The pathspecs that leave out of a package's changes the files its Ignore +# globs match and, unless IgnoreDefaults is false, the default set. +# PowerShell on Linux and macOS replaces a native command's argument that +# holds a '*' with the files it matches, if any. Each of these starts with +# ':(', which no file in a checkout starts with, so git receives it as written. +function Get-PackageExcludes($Package) { + $excludes = @($Package.Ignore | Where-Object { $_ } | + ForEach-Object { ":(exclude,glob)$_" }) + if ($Package.IgnoreDefaults -ne $false) { + # The defaults ignore case, as one language names a folder Tests or + # Examples where another names it tests or examples. + $excludes = @($ReleasePlanDefaultIgnore | + ForEach-Object { ":(exclude,glob,icase)$_" }) + $excludes + } + return $excludes +} + +# The files under the package's folders that changed since the given tag, as +# paths from the repository root, apart from those it ignores. +function Get-ChangedFiles([string]$RepoPath, $Package, [string]$From) { + # A failure is reported below, naming the package, whatever the caller's + # preference. Set for this function only. + $PSNativeCommandUseErrorActionPreference = $false + $pathspecs = @(Get-PackagePaths $Package) + @(Get-PackageExcludes $Package) + $files = @(& git -C $RepoPath diff --name-only $From HEAD -- $pathspecs) + if ($LASTEXITCODE -ne 0) { + throw "git could not list the changes to $($Package.Name) since $From." + } + return $files +} + +# The files that count among a package's changed files. The repository's +# ci/release-plan-filter.ps1 decides when there is one, and otherwise every +# file counts. +function Select-CountedFiles([string]$RepoPath, $Package, [string]$From, + [string[]]$Files) { + $filter = Join-Path $RepoPath 'ci/release-plan-filter.ps1' + if ($Files.Count -eq 0 -or (Test-Path $filter -PathType Leaf) -eq $false) { + return $Files + } + $filter = (Resolve-Path $filter).ProviderPath + return @(& $filter -RepoPath $RepoPath -Package $Package -From $From ` + -Files $Files | Where-Object { $_ }) +} + # Each tox.ini that does not install every package its package depends on, # directly or through another, from this checkout, as a line naming what is # missing. Taken from a package index instead, a dependency is tested at its @@ -224,11 +290,6 @@ function Get-ReleasedVersions([string]$RepoPath, [string]$Name, # from packages.json. Packages that are not published, the examples, are never # in the plan; a change to them is tested but releases nothing. function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { - # 'git diff --quiet' exits 1 to say a package changed. That is an answer, - # not a failure, but a caller that stops on a failing native command, as - # publish-package.ps1 does, stopped there and failed the publish. Set for - # this function only. - $PSNativeCommandUseErrorActionPreference = $false if ($null -eq $Packages) { $Packages = Get-Packages $RepoPath } @@ -254,10 +315,18 @@ function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { $last = $released[-1] $previous[$package.Name] = $last $tag = "$($package.Name)/$last" - & git -C $RepoPath diff --quiet $tag HEAD -- (Get-PackagePaths $package) - if ($LASTEXITCODE -eq 1) { - $reasons[$package.Name] = "changed since $last" + $files = @(Get-ChangedFiles $RepoPath $package $tag) + $counted = @(Select-CountedFiles $RepoPath $package $tag $files) + if ($counted.Count -gt 0) { + $noun = 'files' + if ($counted.Count -eq 1) { + $noun = 'file' + } + $reasons[$package.Name] = + "changed since $last ($($counted.Count) $noun)" } + # A filter may leave an exit code that was an answer, which must not + # read as a failure to the script that called this. $global:LASTEXITCODE = 0 } diff --git a/ci/release-plan-filter.ps1 b/ci/release-plan-filter.ps1 new file mode 100644 index 00000000..a6491a20 --- /dev/null +++ b/ci/release-plan-filter.ps1 @@ -0,0 +1,38 @@ +param ( + [Parameter(Mandatory)][string]$RepoPath, + [Parameter(Mandatory)]$Package, + [Parameter(Mandatory)][string]$From, + [Parameter(Mandatory)][string[]]$Files +) +$ErrorActionPreference = "Stop" + +# Called by the release plan for each package with changed files left once +# the Ignore globs have left theirs out, and returns the files that count. +# A pyproject.toml, setup.py or setup.cfg whose change since $From is only to +# the description, keywords, urls, classifiers, readme or authors does not +# count, which release-plan-filter.py works out. Every other file counts. +# +# Python is required rather than optional, because a job that counted every +# file would plan different releases from a job that could compare them. + +$manifests = 'pyproject.toml', 'setup.py', 'setup.cfg' +$compare = @($Files | Where-Object { (Split-Path -Leaf $_) -in $manifests }) +$counted = @($Files | Where-Object { (Split-Path -Leaf $_) -notin $manifests }) + +if ($compare.Count -gt 0) { + $python = Get-Command python, python3 -CommandType Application ` + -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -eq $python) { + throw "Python is needed to tell whether a change to " + + "$($compare -join ', ') is only to metadata, and none was found." + } + $PSNativeCommandUseErrorActionPreference = $false + $script = Join-Path $PSScriptRoot 'release-plan-filter.py' + $changed = @(& $python.Source -I $script $RepoPath $From @compare) + if ($LASTEXITCODE -ne 0) { + throw "release-plan-filter.py failed for $($Package.Name) since $From." + } + $counted += $changed | Where-Object { $_ } +} + +$counted diff --git a/ci/release-plan-filter.py b/ci/release-plan-filter.py new file mode 100644 index 00000000..babee0b9 --- /dev/null +++ b/ci/release-plan-filter.py @@ -0,0 +1,191 @@ +"""Prints the manifest files whose change since a tag is more than metadata. + +Usage: release-plan-filter.py [ ...] + +Each file is a pyproject.toml, setup.py or setup.cfg, as a path from the +repository root. Its text at the tag and at HEAD is compared with the +description, keywords, urls, classifiers, readme and authors left out, so a +file is printed when anything else changed, and left out when only those +did. A file that is missing on either side, or that cannot be read as what +its name says, is printed, so a release is never skipped by mistake. + +Only the standard library is used, and the same way on every Python this +repository supports, so every job of a publish works out the same plan. +""" + +import ast +import configparser +import os +import re +import subprocess +import sys + +# The setup() arguments and setup.cfg [metadata] options that are metadata. +SETUP_METADATA = frozenset(( + "description", "long_description", "long_description_content_type", + "keywords", "url", "project_urls", "download_url", "classifiers", + "author", "author_email")) + +# The pyproject.toml [project] keys that are metadata. +PROJECT_METADATA = frozenset(( + "description", "keywords", "urls", "classifiers", "readme", "authors")) + +# A table header, such as [project.urls] or [[project.authors]]. +HEADER = re.compile(r"^\[\[?\s*([^\]]+?)\s*\]\]?\s*(#.*)?$") + +# The start of a key and value. The first part of a dotted key is the one +# that says what it is, as urls in urls.Homepage. +ASSIGNMENT = re.compile( + r"^(\"[^\"]*\"|'[^']*'|[A-Za-z0-9_-]+)" + r"(?:\s*\.\s*(?:\"[^\"]*\"|'[^']*'|[A-Za-z0-9_-]+))*\s*=") + + +def text_at(repository, revision, path): + """The file's text at the revision, or None when it is not there.""" + found = subprocess.run( + ["git", "-C", repository, "show", revision + ":" + path], + capture_output=True) + if found.returncode != 0: + return None + return found.stdout.decode("utf-8") + + +class StripSetupMetadata(ast.NodeTransformer): + """Removes the metadata arguments from every call of setup().""" + + def visit_Call(self, node): + self.generic_visit(node) + name = node.func.attr if isinstance(node.func, ast.Attribute) \ + else getattr(node.func, "id", None) + if name == "setup": + node.keywords = [ + keyword for keyword in node.keywords + if keyword.arg not in SETUP_METADATA] + return node + + +def setup_py(text): + # The syntax tree leaves out comments and layout as well. + return ast.dump(StripSetupMetadata().visit(ast.parse(text))) + + +def setup_cfg(text): + parser = configparser.RawConfigParser() + parser.optionxform = str + parser.read_string(text) + return { + section: { + key: value for key, value in parser.items(section) + if section != "metadata" or key not in SETUP_METADATA} + for section in parser.sections()} + + +def toml_lines(text): + """The lines of a TOML file that are not metadata, each with the table + and key it belongs to, leaving out blank lines and comments. + + A small reader rather than tomllib, which Python 3.10 does not have. It + follows tables, keys, values that run over several lines in brackets or + in triple quotes, and comments, which is enough to place every line. + """ + kept = [] + table = "" + key = None + depth = 0 + quote = None + for line in text.splitlines(): + stripped = line.strip() + if quote is None and depth == 0: + if stripped == "" or stripped.startswith("#"): + continue + header = HEADER.match(stripped) + if header: + table = header.group(1).replace(" ", "") + key = None + if not is_metadata(table, None): + kept.append((table, None, stripped)) + continue + assignment = ASSIGNMENT.match(stripped) + if assignment is None: + # Not TOML this reader knows, so it counts unless the whole + # table is metadata. + if not is_metadata(table, "?"): + kept.append((table, "?", stripped)) + continue + key = assignment.group(1).strip("\"'") + value = stripped[assignment.end():] + else: + value = stripped + depth, quote = scan(value, depth, quote) + if not is_metadata(table, key): + kept.append((table, key, stripped)) + return kept + + +def scan(value, depth, quote): + """The bracket depth and the open triple quote after the value.""" + position = 0 + while position < len(value): + if quote is not None: + end = value.find(quote, position) + if end < 0: + return depth, quote + position = end + 3 + quote = None + continue + character = value[position] + if value.startswith('"""', position) or \ + value.startswith("'''", position): + quote = value[position:position + 3] + position += 3 + continue + if character in "\"'": + end = value.find(character, position + 1) + position = len(value) if end < 0 else end + 1 + continue + if character == "#": + break + if character in "[{": + depth += 1 + elif character in "]}": + depth -= 1 + position += 1 + return depth, quote + + +def is_metadata(table, key): + if table == "project": + return key in PROJECT_METADATA + return table in ("project.urls", "project.authors", "project.readme") + + +READERS = { + "setup.py": setup_py, + "setup.cfg": setup_cfg, + "pyproject.toml": toml_lines, +} + + +def counts(repository, tag, path): + """Whether the change to the file since the tag is more than metadata.""" + reader = READERS.get(os.path.basename(path)) + before = text_at(repository, tag, path) + after = text_at(repository, "HEAD", path) + if reader is None or before is None or after is None: + return True + try: + return reader(before) != reader(after) + except (SyntaxError, ValueError, configparser.Error, + UnicodeDecodeError): + return True + + +def main(arguments): + repository, tag = arguments[0], arguments[1] + for path in arguments[2:]: + if counts(repository, tag, path): + print(path) + + +if __name__ == "__main__": + main(sys.argv[1:]) From 0713add4c2fdc32d701c54db2be748f1100b4763 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 18:10:21 +0000 Subject: [PATCH 03/13] Refuse a filter's path it was not given, and IgnoreDefaults as text, as common-ci's shared step does --- ci/packages.Tests.ps1 | 9 +++++++++ ci/packages.ps1 | 23 ++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index ce3cf848..d77c7b4f 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -298,6 +298,15 @@ Describe 'Get-ReleasePlan' { Add-Commit $script:Repo @('device/dd/readme.md') @(Get-ReleasePlan $script:Repo $graph).Name | Should -Be @('dd') } + + It 'refuses IgnoreDefaults given as text' { + $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) + ($graph | Where-Object Name -eq 'dd') | Add-Member IgnoreDefaults 'false' + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/readme.md') + { Get-ReleasePlan $script:Repo $graph } | + Should -Throw '*IgnoreDefaults*true or false*' + } } Describe 'release-plan-filter.ps1' { diff --git a/ci/packages.ps1 b/ci/packages.ps1 index ea785413..7529a333 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -15,6 +15,9 @@ # examples releases nothing. release-plan-filter.ps1 then leaves out a # manifest whose change is only to metadata such as links. These are the # fields and the filter of common-ci's steps/release-plan.ps1. +# +# A submodule in Path or ExtraPaths changes as one path, its own, so any move +# of it counts. Neither the globs nor the filter see inside it. # Reads packages.json. Kept as a function so the tests can give their own list. function Get-Packages([string]$RepoPath) { @@ -57,9 +60,14 @@ $ReleasePlanDefaultIgnore = @( # holds a '*' with the files it matches, if any. Each of these starts with # ':(', which no file in a checkout starts with, so git receives it as written. function Get-PackageExcludes($Package) { + $defaults = $Package.IgnoreDefaults + if ($null -ne $defaults -and $defaults -isnot [bool]) { + throw "$($Package.Name) gives IgnoreDefaults as '$defaults', where " + + "packages.json must give true or false." + } $excludes = @($Package.Ignore | Where-Object { $_ } | ForEach-Object { ":(exclude,glob)$_" }) - if ($Package.IgnoreDefaults -ne $false) { + if ($defaults -ne $false) { # The defaults ignore case, as one language names a folder Tests or # Examples where another names it tests or examples. $excludes = @($ReleasePlanDefaultIgnore | @@ -84,7 +92,9 @@ function Get-ChangedFiles([string]$RepoPath, $Package, [string]$From) { # The files that count among a package's changed files. The repository's # ci/release-plan-filter.ps1 decides when there is one, and otherwise every -# file counts. +# file counts. A path the filter returns that it was not given stops the +# plan, because a filter that rewrote a path would otherwise release a +# package on a file that never changed. function Select-CountedFiles([string]$RepoPath, $Package, [string]$From, [string[]]$Files) { $filter = Join-Path $RepoPath 'ci/release-plan-filter.ps1' @@ -92,8 +102,15 @@ function Select-CountedFiles([string]$RepoPath, $Package, [string]$From, return $Files } $filter = (Resolve-Path $filter).ProviderPath - return @(& $filter -RepoPath $RepoPath -Package $Package -From $From ` + $kept = @(& $filter -RepoPath $RepoPath -Package $Package -From $From ` -Files $Files | Where-Object { $_ }) + foreach ($file in $kept) { + if ($Files -cnotcontains $file) { + throw "ci/release-plan-filter.ps1 returned '$file' for " + + "$($Package.Name), which is not one of the files it was given." + } + } + return $kept } # Each tox.ini that does not install every package its package depends on, From 3232edb01a9cebfb20de528b622be5b45fc1692e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 18:53:51 +0000 Subject: [PATCH 04/13] Count a manifest that is not UTF-8 rather than stopping the plan --- ci/release-plan-filter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/release-plan-filter.py b/ci/release-plan-filter.py index babee0b9..33d474ab 100644 --- a/ci/release-plan-filter.py +++ b/ci/release-plan-filter.py @@ -40,14 +40,14 @@ r"(?:\s*\.\s*(?:\"[^\"]*\"|'[^']*'|[A-Za-z0-9_-]+))*\s*=") -def text_at(repository, revision, path): - """The file's text at the revision, or None when it is not there.""" +def bytes_at(repository, revision, path): + """The file's content at the revision, or None when it is not there.""" found = subprocess.run( ["git", "-C", repository, "show", revision + ":" + path], capture_output=True) if found.returncode != 0: return None - return found.stdout.decode("utf-8") + return found.stdout class StripSetupMetadata(ast.NodeTransformer): @@ -169,12 +169,12 @@ def is_metadata(table, key): def counts(repository, tag, path): """Whether the change to the file since the tag is more than metadata.""" reader = READERS.get(os.path.basename(path)) - before = text_at(repository, tag, path) - after = text_at(repository, "HEAD", path) + before = bytes_at(repository, tag, path) + after = bytes_at(repository, "HEAD", path) if reader is None or before is None or after is None: return True try: - return reader(before) != reader(after) + return reader(before.decode("utf-8")) != reader(after.decode("utf-8")) except (SyntaxError, ValueError, configparser.Error, UnicodeDecodeError): return True From d3ce8df0b7908d2c287fca352ffa38839bf0505a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 18:54:35 +0000 Subject: [PATCH 05/13] Test that a manifest that is not UTF-8 counts --- ci/packages.Tests.ps1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index d77c7b4f..5038686e 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -474,6 +474,14 @@ Source = "https://github.com/51Degrees/51degrees-python" Set-Files $script:Repo @{ 'device/dd/setup.py' = 'setup(' } (Get-DdPlan $script:Repo).Count | Should -Be 1 } + + It 'releases for a manifest that is not UTF-8' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + [IO.File]::WriteAllBytes((Join-Path $script:Repo 'device/dd/setup.py'), + [byte[]](0x23, 0xff, 0xfe, 0x0a)) + Set-Files $script:Repo @{} + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } } Describe 'Get-BuildPlan' { From 8637ebde5941e69bee1405be282f3a1ec60b57a6 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 19:48:04 +0000 Subject: [PATCH 06/13] Release no package for a change only to its tox.ini A package's tox.ini sets up its test environments and is not part of what is published, so a change to it alone no longer releases the package. Each package with a tox.ini ignores it in packages.json, and the tests cover the real package list. --- ci/packages.Tests.ps1 | 38 ++++++++++++++++++++++++++++++++++++++ ci/packages.json | 15 +++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 5038686e..61653487 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -555,6 +555,44 @@ Describe 'the real packages.json' { Get-ToxGaps (Split-Path $PSScriptRoot) $script:Real | Should -BeNullOrEmpty } + + Context 'a change to a package''s tox.ini' { + BeforeAll { + $published = @($script:Real | Where-Object Publish -eq $true) + $script:RealTags = @($published | ForEach-Object { "$($_.Name)/4.5.1" }) + $script:RealFiles = @($script:Real | ForEach-Object { + "$($_.Path)/src/a.py" + "$($_.Path)/tox.ini" + }) + } + + It 'releases nothing, whichever package it sits beside' { + $script:Repo = New-Repo $script:RealFiles $script:RealTags + Add-Commit $script:Repo ($script:Real | ForEach-Object { "$($_.Path)/tox.ini" }) + @(Get-ReleasePlan $script:Repo $script:Real).Count | Should -Be 0 + } + + It 'still releases a package for a change to its source' { + $script:Repo = New-Repo $script:RealFiles $script:RealTags + Add-Commit $script:Repo @( + 'pipeline/fiftyone_pipeline_core/tox.ini', + 'pipeline/fiftyone_pipeline_core/src/a.py') + $plan = @(Get-ReleasePlan $script:Repo $script:Real) + $plan.Name | Should -Contain 'fiftyone_pipeline_core' + ($plan | Where-Object Name -eq 'fiftyone_pipeline_core').Reason | + Should -Be 'changed since 4.5.1 (1 file)' + } + + It 'is ignored by every package that has a tox.ini' { + $root = Split-Path $PSScriptRoot + $beside = @($script:Real | Where-Object { + Test-Path (Join-Path $root "$($_.Path)/tox.ini") -PathType Leaf }) + $beside.Count | Should -BeGreaterThan 10 + foreach ($package in $beside) { + $package.Ignore | Should -Contain '**/tox.ini' -Because $package.Name + } + } + } } } diff --git a/ci/packages.json b/ci/packages.json index deb06ce9..1c9afb8a 100644 --- a/ci/packages.json +++ b/ci/packages.json @@ -3,6 +3,7 @@ "Name": "fiftyone_pipeline_core", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_core", + "Ignore": ["**/tox.ini"], "DependsOn": [], "Publish": true }, @@ -10,6 +11,7 @@ "Name": "fiftyone_pipeline_engines", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_engines", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_core"], "Publish": true }, @@ -17,6 +19,7 @@ "Name": "fiftyone_pipeline_engines_fiftyone", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_engines_fiftyone", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_engines"], "Publish": true }, @@ -24,6 +27,7 @@ "Name": "fiftyone_pipeline_cloudrequestengine", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_cloudrequestengine", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_engines"], "Publish": true }, @@ -31,6 +35,7 @@ "Name": "fiftyone_pipeline_translation", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_translation", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_core"], "Publish": true }, @@ -38,6 +43,7 @@ "Name": "fiftyone_pipeline_did", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_did", + "Ignore": ["**/tox.ini"], "ExtraPaths": ["pipeline/owid-python"], "DependsOn": [], "Publish": true @@ -69,6 +75,7 @@ "Name": "fiftyone_devicedetection_cloud", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_cloud", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_devicedetection_shared"], "Publish": true }, @@ -76,6 +83,7 @@ "Name": "fiftyone_devicedetection_onpremise", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_onpremise", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_shared", "fiftyone_pipeline_engines_fiftyone" @@ -98,6 +106,7 @@ "Name": "fiftyone_devicedetection_examples", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_cloud", "fiftyone_devicedetection" @@ -108,6 +117,7 @@ "Name": "fiftyone_location", "Product": "location", "Path": "location", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -119,6 +129,7 @@ "Name": "fiftyone_ipintelligence_shared", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_shared", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -132,6 +143,7 @@ "Name": "fiftyone_ipintelligence_cloud", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_cloud", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_ipintelligence_shared"], "Publish": true, "FirstVersion": "4.5.0" @@ -140,6 +152,7 @@ "Name": "fiftyone_ipintelligence_onpremise", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_onpremise", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_pipeline_engines_fiftyone" @@ -152,6 +165,7 @@ "Name": "fiftyone_ipintelligence_translation", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_translation", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_translation" @@ -175,6 +189,7 @@ "Name": "fiftyone_ipintelligence_examples", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence", "fiftyone_ipintelligence_translation" From 3668b675df78b10b2241adaa7703405ed76fad93 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 20:04:25 +0000 Subject: [PATCH 07/13] Ignore tox.ini in every package, including those without one yet --- ci/packages.Tests.ps1 | 8 ++------ ci/packages.json | 4 ++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 61653487..d4d41d5d 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -583,12 +583,8 @@ Describe 'the real packages.json' { Should -Be 'changed since 4.5.1 (1 file)' } - It 'is ignored by every package that has a tox.ini' { - $root = Split-Path $PSScriptRoot - $beside = @($script:Real | Where-Object { - Test-Path (Join-Path $root "$($_.Path)/tox.ini") -PathType Leaf }) - $beside.Count | Should -BeGreaterThan 10 - foreach ($package in $beside) { + It 'is ignored by every package, so one added later counts for nothing' { + foreach ($package in $script:Real) { $package.Ignore | Should -Contain '**/tox.ini' -Because $package.Name } } diff --git a/ci/packages.json b/ci/packages.json index 1c9afb8a..d9dcd87d 100644 --- a/ci/packages.json +++ b/ci/packages.json @@ -52,6 +52,7 @@ "Name": "fiftyone_pipeline_examples", "Product": "pipeline", "Path": "pipeline/examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -63,6 +64,7 @@ "Name": "fiftyone_devicedetection_shared", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_shared", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -95,6 +97,7 @@ "Name": "fiftyone_devicedetection", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_shared", "fiftyone_devicedetection_cloud", @@ -177,6 +180,7 @@ "Name": "fiftyone_ipintelligence", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_ipintelligence_cloud", From e2220a188346e5e416bac2a892afdc1c87094d86 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 17:37:26 +0000 Subject: [PATCH 08/13] Test that a change only to the maintainers releases nothing --- ci/packages.Tests.ps1 | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index d4d41d5d..243d87a6 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -453,6 +453,32 @@ Source = "https://github.com/51Degrees/51degrees-python" (Get-DdPlan $script:Repo).Count | Should -Be 0 } + It 'does not release for a change only to the maintainers in ' -ForEach @( + @{ Case = 'setup.py'; File = 'setup.py'; Start = 'SetupPy' + Was = 'author_email="engineering@51degrees.com",' + Changed = "author_email=`"engineering@51degrees.com`",`n maintainer=`"51Degrees`",`n maintainer_email=`"support@51degrees.com`"," } + @{ Case = 'setup.cfg'; File = 'setup.cfg'; Start = 'SetupCfg' + Was = "name = dd`n" + Changed = "name = dd`nmaintainer = 51Degrees`nmaintainer_email = support@51degrees.com`n" } + @{ Case = 'a pyproject.toml key'; File = 'pyproject.toml'; Start = 'Pyproject' + Was = 'dependencies = ' + Changed = "maintainers = [{ name = `"51Degrees`" }]`ndependencies = " } + @{ Case = 'a pyproject.toml table'; File = 'pyproject.toml'; Start = 'Pyproject' + Was = '[project.urls]' + Changed = "[[project.maintainers]]`nname = `"51Degrees`"`n`n[project.urls]" } + ) { + $text = @{ + SetupPy = $script:SetupPy + SetupCfg = $script:SetupCfg + Pyproject = $script:Pyproject + }[$Start] + $changed = $text.Replace($Was, $Changed) + $changed | Should -Not -Be $text + $script:Repo = New-FilterRepo @{ "device/dd/$File" = $text } + Set-Files $script:Repo @{ "device/dd/$File" = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + It 'releases for a pyproject.toml change to ' -ForEach @( @{ Was = '["fiftyone_pipeline_core"]'; Changed = '["fiftyone_pipeline_core>=4.5"]' } @{ Was = '"setuptools>=42"'; Changed = '"setuptools>=61"' } From 766fe11656bcfeab48efae7ea66c8395677806d7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 17:37:27 +0000 Subject: [PATCH 09/13] Leave the maintainers out of the manifest metadata, as the authors are --- AGENTS.md | 11 ++++++----- ci/release-plan-filter.py | 17 ++++++++++------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8a362623..3bec0789 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -273,11 +273,12 @@ last tag, or when a package it depends on is released. Documentation `LICENSE` files do not count, nor do files a package's `Ignore` globs in `ci/packages.json` match, unless it sets `IgnoreDefaults` to false. A `setup.py`, `setup.cfg` or `pyproject.toml` whose change is only to the -description, keywords, urls, classifiers, readme or authors does not count -either, which `ci/release-plan-filter.ps1` works out with Python. The -examples packages are tested but never published. common-ci's publish carries one version; here it is only a label, -and the release plan in `ci/packages.ps1` decides what is built, published -and tagged. Never push a plain `4.5.x` tag: nothing here reads one. +description, keywords, urls, classifiers, readme, authors or maintainers does +not count either, which `ci/release-plan-filter.ps1` works out with Python. +The examples packages are tested but never published. common-ci's publish +carries one version, which here is only a label, and the release plan in +`ci/packages.ps1` decides what is built, published and tagged. Never push a +plain `4.5.x` tag, because nothing here reads one. `ci/packages.json` is the list of packages and what each depends on. Add a package there, with a `FirstVersion` if it has never been released, or it is diff --git a/ci/release-plan-filter.py b/ci/release-plan-filter.py index 33d474ab..94191c90 100644 --- a/ci/release-plan-filter.py +++ b/ci/release-plan-filter.py @@ -4,9 +4,9 @@ Each file is a pyproject.toml, setup.py or setup.cfg, as a path from the repository root. Its text at the tag and at HEAD is compared with the -description, keywords, urls, classifiers, readme and authors left out, so a -file is printed when anything else changed, and left out when only those -did. A file that is missing on either side, or that cannot be read as what +description, keywords, urls, classifiers, readme, authors and maintainers +left out, so a file is printed when anything else changed, and left out when +only those did. A file that is missing on either side, or that cannot be read as what its name says, is printed, so a release is never skipped by mistake. Only the standard library is used, and the same way on every Python this @@ -24,13 +24,14 @@ SETUP_METADATA = frozenset(( "description", "long_description", "long_description_content_type", "keywords", "url", "project_urls", "download_url", "classifiers", - "author", "author_email")) + "author", "author_email", "maintainer", "maintainer_email")) # The pyproject.toml [project] keys that are metadata. PROJECT_METADATA = frozenset(( - "description", "keywords", "urls", "classifiers", "readme", "authors")) + "description", "keywords", "urls", "classifiers", "readme", "authors", + "maintainers")) -# A table header, such as [project.urls] or [[project.authors]]. +# A table header, such as [project.urls] or [[project.maintainers]]. HEADER = re.compile(r"^\[\[?\s*([^\]]+?)\s*\]\]?\s*(#.*)?$") # The start of a key and value. The first part of a dotted key is the one @@ -156,7 +157,9 @@ def scan(value, depth, quote): def is_metadata(table, key): if table == "project": return key in PROJECT_METADATA - return table in ("project.urls", "project.authors", "project.readme") + return table in ( + "project.urls", "project.authors", "project.maintainers", + "project.readme") READERS = { From 642d492a91309a7e513da21d5bf761ddd3319013 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 17:45:45 +0000 Subject: [PATCH 10/13] Say that the release plan comes from common-ci, and that packages.ps1 also gives the build plan --- AGENTS.md | 7 ++++--- ci/packages.ps1 | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3bec0789..044c5e15 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -276,9 +276,10 @@ last tag, or when a package it depends on is released. Documentation description, keywords, urls, classifiers, readme, authors or maintainers does not count either, which `ci/release-plan-filter.ps1` works out with Python. The examples packages are tested but never published. common-ci's publish -carries one version, which here is only a label, and the release plan in -`ci/packages.ps1` decides what is built, published and tagged. Never push a -plain `4.5.x` tag, because nothing here reads one. +carries one version, which here is only a label, and the release plan +decides what is built, published and tagged. The release plan is common-ci's +`steps/release-plan.ps1`, which `ci/packages.ps1` loads, and common-ci tests +it. Never push a plain `4.5.x` tag, because nothing here reads one. `ci/packages.json` is the list of packages and what each depends on. Add a package there, with a `FirstVersion` if it has never been released, or it is diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 7a2f99ee..40609f0b 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -1,5 +1,6 @@ # Shared by the scripts in this folder, which dot-source it. Works out which -# products a pull request touches, and checks each package's tox.ini. +# products a pull request touches, what a build of HEAD makes, and checks each +# package's tox.ini. # # Which packages a publish releases, and at which version, is worked out by # common-ci's steps/release-plan.ps1 from ci/packages.json. common-ci runs From d292d72081b13cac239b812d03cb4535c1091126 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 18:00:31 +0000 Subject: [PATCH 11/13] Give the dispatch tests' common-ci folder the release plan, as a real checkout has --- ci/dispatch.Tests.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/dispatch.Tests.ps1 b/ci/dispatch.Tests.ps1 index 3d75e0a2..c4ed1b59 100644 --- a/ci/dispatch.Tests.ps1 +++ b/ci/dispatch.Tests.ps1 @@ -17,6 +17,9 @@ BeforeAll { if ((Test-Path $script:RunScript) -eq $false) { throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it" } + # ci/packages.ps1 loads the release plan from the common-ci checkout + # that holds this repository's folder. + $script:ReleasePlan = Join-Path $commonCi 'steps' 'release-plan.ps1' # A repository with packages.json naming one package in each product. function New-ProductRepo { @@ -46,7 +49,7 @@ BeforeAll { function New-CommonCi { $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) $steps = New-Item -ItemType Directory -Force (Join-Path $path 'steps') - Copy-Item $script:RunScript $steps + Copy-Item $script:RunScript, $script:ReleasePlan $steps return $path } } From ec8b2fdd94f4f31fbda1dfad1ad31227fbdca734 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 19:11:38 +0000 Subject: [PATCH 12/13] Name the maintainers among the fields the filter leaves out, in the hook's comment --- ci/release-plan-filter.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/release-plan-filter.ps1 b/ci/release-plan-filter.ps1 index a6491a20..dcd316aa 100644 --- a/ci/release-plan-filter.ps1 +++ b/ci/release-plan-filter.ps1 @@ -9,8 +9,9 @@ $ErrorActionPreference = "Stop" # Called by the release plan for each package with changed files left once # the Ignore globs have left theirs out, and returns the files that count. # A pyproject.toml, setup.py or setup.cfg whose change since $From is only to -# the description, keywords, urls, classifiers, readme or authors does not -# count, which release-plan-filter.py works out. Every other file counts. +# the description, keywords, urls, classifiers, readme, authors or maintainers +# does not count, which release-plan-filter.py works out. Every other file +# counts. # # Python is required rather than optional, because a job that counted every # file would plan different releases from a job that could compare them. From 742f25689172daa04270b4b0e63856da7e8cc8e1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Fri, 13 Nov 2026 19:13:38 +0000 Subject: [PATCH 13/13] Copy common-ci's whole steps folder into the dispatch tests' stand-in checkout --- ci/dispatch.Tests.ps1 | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/dispatch.Tests.ps1 b/ci/dispatch.Tests.ps1 index c4ed1b59..352e667b 100644 --- a/ci/dispatch.Tests.ps1 +++ b/ci/dispatch.Tests.ps1 @@ -2,8 +2,8 @@ # # Each test builds a throwaway repository with two products, 'alpha' and # 'beta', whose ci/ scripts are written by the test, and runs the dispatcher -# from a folder laid out like a common-ci checkout, with common-ci's real -# steps/run-script.ps1, because that is how common-ci calls it. +# from a folder laid out like a common-ci checkout, with a copy of common-ci's +# real steps folder, because that is how common-ci calls it. BeforeAll { . (Join-Path $PSScriptRoot 'dispatch.ps1') @@ -17,9 +17,10 @@ BeforeAll { if ((Test-Path $script:RunScript) -eq $false) { throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it" } - # ci/packages.ps1 loads the release plan from the common-ci checkout - # that holds this repository's folder. - $script:ReleasePlan = Join-Path $commonCi 'steps' 'release-plan.ps1' + # The whole steps folder is copied, because ci/packages.ps1 loads the + # release plan from the common-ci checkout that holds this repository's + # folder, and the release plan may load other steps beside it. + $script:Steps = Join-Path $commonCi 'steps' # A repository with packages.json naming one package in each product. function New-ProductRepo { @@ -48,8 +49,8 @@ BeforeAll { # A folder laid out like a common-ci checkout, which the dispatcher runs in. function New-CommonCi { $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - $steps = New-Item -ItemType Directory -Force (Join-Path $path 'steps') - Copy-Item $script:RunScript, $script:ReleasePlan $steps + $null = New-Item -ItemType Directory -Force $path + Copy-Item -Recurse $script:Steps (Join-Path $path 'steps') return $path } }