diff --git a/AGENTS.md b/AGENTS.md index 39b710d7..3bec0789 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -267,11 +267,18 @@ touches all of them ([ci/packages.ps1](ci/packages.ps1), ## Versions are per package Every package is versioned and tagged on its own, `/`. A -package is released when its folders changed since its last tag, or when a -package it depends on is released. The examples packages are tested but never -published. common-ci's publish carries one version; here it is only a label, -and the release plan in `ci/packages.ps1` decides what is built, published -and tagged. Never push a plain `4.5.x` tag: nothing here reads one. +package is released when a file that counts in its folders changed since its +last tag, or when a package it depends on is released. Documentation +(`*.md`, `docs/`), tests (`tests/`, `test/`), examples (`examples/`) and +`LICENSE` files do not count, nor do files a package's `Ignore` globs in +`ci/packages.json` match, unless it sets `IgnoreDefaults` to false. A +`setup.py`, `setup.cfg` or `pyproject.toml` whose change is only to the +description, keywords, urls, classifiers, readme, authors or maintainers does +not count either, which `ci/release-plan-filter.ps1` works out with Python. +The examples packages are tested but never published. common-ci's publish +carries one version, which here is only a label, and the release plan in +`ci/packages.ps1` decides what is built, published and tagged. Never push a +plain `4.5.x` tag, because nothing here reads one. `ci/packages.json` is the list of packages and what each depends on. Add a package there, with a `FirstVersion` if it has never been released, or it is diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 766d1380..243d87a6 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -247,6 +247,267 @@ Describe 'Get-ReleasePlan' { $script:Repo = New-Repo @('p/new/a.py') @() { Get-ReleasePlan $script:Repo $graph } | Should -Throw '*FirstVersion*' } + + It 'says how many files counted' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/a.py', 'device/dd/b.py') + (@(Get-ReleasePlan $script:Repo $script:Graph) | + Where-Object Name -eq 'dd').Reason | + Should -Be 'changed since 4.5.96 (2 files)' + } + + It 'releases nothing for a change only to documentation, tests or examples' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @( + 'device/dd/readme.md', 'device/dd/docs/guide.txt', + 'device/dd/tests/test_a.py', 'device/dd/test/a.py', + 'device/dd/examples/a.py', 'device/dd/LICENSE', + 'pipeline/core/sub/notes.md') + @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 + } + + It 'ignores documentation, tests and examples whatever their case' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @( + 'device/dd/README.MD', 'device/dd/Tests/a.py', + 'device/dd/Examples/a.py') + @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 + } + + It 'releases for the code changed beside the documentation' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/readme.md', 'device/dd/a.py') + $plan = @(Get-ReleasePlan $script:Repo $script:Graph) + $plan.Name | Should -Be @('dd') + $plan[0].Reason | Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'leaves out the files a package''s Ignore globs match' { + $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) + ($graph | Where-Object Name -eq 'dd') | + Add-Member Ignore @('device/dd/tox.ini') + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/tox.ini') + @(Get-ReleasePlan $script:Repo $graph).Count | Should -Be 0 + } + + It 'counts documentation for a package that turns the defaults off' { + $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) + ($graph | Where-Object Name -eq 'dd') | Add-Member IgnoreDefaults $false + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/readme.md') + @(Get-ReleasePlan $script:Repo $graph).Name | Should -Be @('dd') + } + + It 'refuses IgnoreDefaults given as text' { + $graph = @($script:Graph | ForEach-Object { $_.PSObject.Copy() }) + ($graph | Where-Object Name -eq 'dd') | Add-Member IgnoreDefaults 'false' + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/readme.md') + { Get-ReleasePlan $script:Repo $graph } | + Should -Throw '*IgnoreDefaults*true or false*' + } +} + +Describe 'release-plan-filter.ps1' { + BeforeAll { + # A setup.py, setup.cfg and pyproject.toml like the ones here, with + # the metadata the filter leaves out and settings it does not. Line + # endings are made LF, whatever the checkout gave this file. + $script:SetupPy = @' +import setuptools + +setuptools.setup( + name="dd", + version=read("version.txt"), + author="51Degrees Engineering", + author_email="engineering@51degrees.com", + project_urls={"Source": "https://github.com/51Degrees/51degrees-python"}, + description=("Device detection."), + keywords=["51degrees", "device-detection"], + long_description=read("readme.md"), + long_description_content_type='text/markdown', + python_requires=">=3.10", + install_requires=["fiftyone_pipeline_core"], + classifiers=["Programming Language :: Python"], +) +'@.Replace("`r`n", "`n") + $script:SetupCfg = @' +[metadata] +name = dd +description = Device detection. +url = https://github.com/51Degrees/51degrees-python + +[options] +install_requires = + fiftyone_pipeline_core +'@.Replace("`r`n", "`n") + "`n" + $script:Pyproject = @' +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "dd" +description = "Device detection." +keywords = [ + "51degrees", + "device-detection", +] +dependencies = ["fiftyone_pipeline_core"] + +[project.urls] +Source = "https://github.com/51Degrees/51degrees-python" +'@.Replace("`r`n", "`n") + "`n" + + # A repository with the given files, holding the filter as this + # repository does, with the tags of the small graph and the tag of + # dd on the commit that adds the files. + function New-FilterRepo([hashtable]$Files) { + $repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + $null = New-Item -ItemType Directory -Force (Join-Path $repo 'ci') + foreach ($name in 'release-plan-filter.ps1', 'release-plan-filter.py') { + Copy-Item (Join-Path $PSScriptRoot $name) (Join-Path $repo 'ci') + } + Set-Files $repo $Files + git -C $repo tag -f 'dd/4.5.96' | Out-Null + return $repo + } + + # Commits the files with exactly the text given. + function Set-Files([string]$Repo, [hashtable]$Files) { + foreach ($file in $Files.Keys) { + $full = Join-Path $Repo $file + $null = New-Item -ItemType Directory -Force (Split-Path $full) + [IO.File]::WriteAllText($full, $Files[$file]) + } + git -C $Repo add -A + git -C $Repo -c user.name=test -c user.email=test@example.com ` + commit --quiet --allow-empty -m 'Change' + } + + function Get-DdPlan([string]$Repo) { + return @(Get-ReleasePlan $Repo $script:Graph | + Where-Object Name -eq 'dd') + } + } + + It 'does not release for a setup.py change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + $changed = $script:SetupPy + $changed = $changed.Replace('Device detection.', 'Device detection for Python.') + $changed = $changed.Replace('"device-detection"]', '"device-detection", "user-agent"]') + $changed = $changed.Replace('51degrees-python"}', '51degrees-python", "Docs": "x"}') + $changed = $changed.Replace('51Degrees Engineering', '51Degrees') + $changed = $changed.Replace('"Programming Language :: Python"]', + '"Programming Language :: Python", "Typing :: Typed"]') + $changed = $changed.Replace("'text/markdown'", "'text/x-rst'") + Set-Files $script:Repo @{ 'device/dd/setup.py' = $changed + "# A note.`n" } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a setup.py change to anything else' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + $changed = $script:SetupPy.Replace('["fiftyone_pipeline_core"]', + '["fiftyone_pipeline_core>=4.5.40"]') + Set-Files $script:Repo @{ 'device/dd/setup.py' = $changed } + $plan = Get-DdPlan $script:Repo + $plan.Count | Should -Be 1 + $plan[0].Reason | Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'counts the code changed beside a setup.py change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + Set-Files $script:Repo @{ + 'device/dd/setup.py' = $script:SetupPy.Replace( + 'Device detection.', 'Device detection for Python.') + 'device/dd/src/a.py' = 'print(1)' + } + (Get-DdPlan $script:Repo)[0].Reason | + Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'does not release for a setup.cfg change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.cfg' = $script:SetupCfg } + $changed = $script:SetupCfg.Replace('Device detection.', 'Device detection for Python.') + $changed = $changed.Replace('51degrees-python', '51degrees-python/tree/main') + Set-Files $script:Repo @{ 'device/dd/setup.cfg' = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a setup.cfg change to its options' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.cfg' = $script:SetupCfg } + Set-Files $script:Repo @{ 'device/dd/setup.cfg' = + $script:SetupCfg + " fiftyone_pipeline_engines`n" } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'does not release for a pyproject.toml change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/pyproject.toml' = $script:Pyproject } + $changed = $script:Pyproject.Replace('"Device detection."', '"Device detection for Python."') + $changed = $changed.Replace(" `"device-detection`",`n", + " `"device-detection`",`n `"user-agent`",`n") + $changed = $changed.Replace('Source = ', "Docs = `"x`"`nSource = ") + $changed += "`n[[project.authors]]`nname = `"51Degrees`"`n" + Set-Files $script:Repo @{ 'device/dd/pyproject.toml' = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'does not release for a change only to the maintainers in ' -ForEach @( + @{ Case = 'setup.py'; File = 'setup.py'; Start = 'SetupPy' + Was = 'author_email="engineering@51degrees.com",' + Changed = "author_email=`"engineering@51degrees.com`",`n maintainer=`"51Degrees`",`n maintainer_email=`"support@51degrees.com`"," } + @{ Case = 'setup.cfg'; File = 'setup.cfg'; Start = 'SetupCfg' + Was = "name = dd`n" + Changed = "name = dd`nmaintainer = 51Degrees`nmaintainer_email = support@51degrees.com`n" } + @{ Case = 'a pyproject.toml key'; File = 'pyproject.toml'; Start = 'Pyproject' + Was = 'dependencies = ' + Changed = "maintainers = [{ name = `"51Degrees`" }]`ndependencies = " } + @{ Case = 'a pyproject.toml table'; File = 'pyproject.toml'; Start = 'Pyproject' + Was = '[project.urls]' + Changed = "[[project.maintainers]]`nname = `"51Degrees`"`n`n[project.urls]" } + ) { + $text = @{ + SetupPy = $script:SetupPy + SetupCfg = $script:SetupCfg + Pyproject = $script:Pyproject + }[$Start] + $changed = $text.Replace($Was, $Changed) + $changed | Should -Not -Be $text + $script:Repo = New-FilterRepo @{ "device/dd/$File" = $text } + Set-Files $script:Repo @{ "device/dd/$File" = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a pyproject.toml change to ' -ForEach @( + @{ Was = '["fiftyone_pipeline_core"]'; Changed = '["fiftyone_pipeline_core>=4.5"]' } + @{ Was = '"setuptools>=42"'; Changed = '"setuptools>=61"' } + ) { + $script:Repo = New-FilterRepo @{ 'device/dd/pyproject.toml' = $script:Pyproject } + Set-Files $script:Repo @{ 'device/dd/pyproject.toml' = + $script:Pyproject.Replace($Was, $Changed) } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest that is new since the last tag' { + $script:Repo = New-FilterRepo @{} + Set-Files $script:Repo @{ 'device/dd/setup.py' = $script:SetupPy } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest it cannot read' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + Set-Files $script:Repo @{ 'device/dd/setup.py' = 'setup(' } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest that is not UTF-8' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + [IO.File]::WriteAllBytes((Join-Path $script:Repo 'device/dd/setup.py'), + [byte[]](0x23, 0xff, 0xfe, 0x0a)) + Set-Files $script:Repo @{} + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } } Describe 'Get-BuildPlan' { @@ -320,6 +581,40 @@ Describe 'the real packages.json' { Get-ToxGaps (Split-Path $PSScriptRoot) $script:Real | Should -BeNullOrEmpty } + + Context 'a change to a package''s tox.ini' { + BeforeAll { + $published = @($script:Real | Where-Object Publish -eq $true) + $script:RealTags = @($published | ForEach-Object { "$($_.Name)/4.5.1" }) + $script:RealFiles = @($script:Real | ForEach-Object { + "$($_.Path)/src/a.py" + "$($_.Path)/tox.ini" + }) + } + + It 'releases nothing, whichever package it sits beside' { + $script:Repo = New-Repo $script:RealFiles $script:RealTags + Add-Commit $script:Repo ($script:Real | ForEach-Object { "$($_.Path)/tox.ini" }) + @(Get-ReleasePlan $script:Repo $script:Real).Count | Should -Be 0 + } + + It 'still releases a package for a change to its source' { + $script:Repo = New-Repo $script:RealFiles $script:RealTags + Add-Commit $script:Repo @( + 'pipeline/fiftyone_pipeline_core/tox.ini', + 'pipeline/fiftyone_pipeline_core/src/a.py') + $plan = @(Get-ReleasePlan $script:Repo $script:Real) + $plan.Name | Should -Contain 'fiftyone_pipeline_core' + ($plan | Where-Object Name -eq 'fiftyone_pipeline_core').Reason | + Should -Be 'changed since 4.5.1 (1 file)' + } + + It 'is ignored by every package, so one added later counts for nothing' { + foreach ($package in $script:Real) { + $package.Ignore | Should -Contain '**/tox.ini' -Because $package.Name + } + } + } } } diff --git a/ci/packages.json b/ci/packages.json index deb06ce9..d9dcd87d 100644 --- a/ci/packages.json +++ b/ci/packages.json @@ -3,6 +3,7 @@ "Name": "fiftyone_pipeline_core", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_core", + "Ignore": ["**/tox.ini"], "DependsOn": [], "Publish": true }, @@ -10,6 +11,7 @@ "Name": "fiftyone_pipeline_engines", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_engines", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_core"], "Publish": true }, @@ -17,6 +19,7 @@ "Name": "fiftyone_pipeline_engines_fiftyone", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_engines_fiftyone", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_engines"], "Publish": true }, @@ -24,6 +27,7 @@ "Name": "fiftyone_pipeline_cloudrequestengine", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_cloudrequestengine", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_engines"], "Publish": true }, @@ -31,6 +35,7 @@ "Name": "fiftyone_pipeline_translation", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_translation", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_core"], "Publish": true }, @@ -38,6 +43,7 @@ "Name": "fiftyone_pipeline_did", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_did", + "Ignore": ["**/tox.ini"], "ExtraPaths": ["pipeline/owid-python"], "DependsOn": [], "Publish": true @@ -46,6 +52,7 @@ "Name": "fiftyone_pipeline_examples", "Product": "pipeline", "Path": "pipeline/examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -57,6 +64,7 @@ "Name": "fiftyone_devicedetection_shared", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_shared", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -69,6 +77,7 @@ "Name": "fiftyone_devicedetection_cloud", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_cloud", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_devicedetection_shared"], "Publish": true }, @@ -76,6 +85,7 @@ "Name": "fiftyone_devicedetection_onpremise", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_onpremise", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_shared", "fiftyone_pipeline_engines_fiftyone" @@ -87,6 +97,7 @@ "Name": "fiftyone_devicedetection", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_shared", "fiftyone_devicedetection_cloud", @@ -98,6 +109,7 @@ "Name": "fiftyone_devicedetection_examples", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_cloud", "fiftyone_devicedetection" @@ -108,6 +120,7 @@ "Name": "fiftyone_location", "Product": "location", "Path": "location", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -119,6 +132,7 @@ "Name": "fiftyone_ipintelligence_shared", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_shared", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -132,6 +146,7 @@ "Name": "fiftyone_ipintelligence_cloud", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_cloud", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_ipintelligence_shared"], "Publish": true, "FirstVersion": "4.5.0" @@ -140,6 +155,7 @@ "Name": "fiftyone_ipintelligence_onpremise", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_onpremise", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_pipeline_engines_fiftyone" @@ -152,6 +168,7 @@ "Name": "fiftyone_ipintelligence_translation", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_translation", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_translation" @@ -163,6 +180,7 @@ "Name": "fiftyone_ipintelligence", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_ipintelligence_cloud", @@ -175,6 +193,7 @@ "Name": "fiftyone_ipintelligence_examples", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence", "fiftyone_ipintelligence_translation" diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 429c43ff..7529a333 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -4,10 +4,20 @@ # which version. # # Each package has its own version, tagged '/'. A package is -# released when a file under its folder changed since its last tag, or when a -# package it depends on is being released, because its requirement on that -# package moves. This is the scheme in 51Degrees/rust#205, which the per -# language repositories follow. +# released when a file that counts under its folders changed since its last +# tag, or when a package it depends on is being released, because its +# requirement on that package moves. This is the scheme in 51Degrees/rust#205, +# which the per language repositories follow. +# +# Every file counts except those matched by the package's Ignore globs in +# packages.json and, unless its IgnoreDefaults is false, by +# $ReleasePlanDefaultIgnore, so a change only to documentation, tests or +# examples releases nothing. release-plan-filter.ps1 then leaves out a +# manifest whose change is only to metadata such as links. These are the +# fields and the filter of common-ci's steps/release-plan.ps1. +# +# A submodule in Path or ExtraPaths changes as one path, its own, so any move +# of it counts. Neither the globs nor the filter see inside it. # Reads packages.json. Kept as a function so the tests can give their own list. function Get-Packages([string]$RepoPath) { @@ -30,6 +40,79 @@ function Test-PathUnder([string]$Path, [string]$Folder) { return $Path -eq $Folder -or $Path.StartsWith("$Folder/") } +# The files whose changes do not release a package, being documentation, +# tests, examples and LICENSE files wherever they sit and whatever their case, +# unless the package sets IgnoreDefaults to false. These are git glob +# pathspecs from the repository root, where '**/' matches any number of +# folders, none included. +$ReleasePlanDefaultIgnore = @( + '**/*.md' + '**/docs/**' + '**/tests/**' + '**/test/**' + '**/examples/**' + '**/LICENSE*' +) + +# The pathspecs that leave out of a package's changes the files its Ignore +# globs match and, unless IgnoreDefaults is false, the default set. +# PowerShell on Linux and macOS replaces a native command's argument that +# holds a '*' with the files it matches, if any. Each of these starts with +# ':(', which no file in a checkout starts with, so git receives it as written. +function Get-PackageExcludes($Package) { + $defaults = $Package.IgnoreDefaults + if ($null -ne $defaults -and $defaults -isnot [bool]) { + throw "$($Package.Name) gives IgnoreDefaults as '$defaults', where " + + "packages.json must give true or false." + } + $excludes = @($Package.Ignore | Where-Object { $_ } | + ForEach-Object { ":(exclude,glob)$_" }) + if ($defaults -ne $false) { + # The defaults ignore case, as one language names a folder Tests or + # Examples where another names it tests or examples. + $excludes = @($ReleasePlanDefaultIgnore | + ForEach-Object { ":(exclude,glob,icase)$_" }) + $excludes + } + return $excludes +} + +# The files under the package's folders that changed since the given tag, as +# paths from the repository root, apart from those it ignores. +function Get-ChangedFiles([string]$RepoPath, $Package, [string]$From) { + # A failure is reported below, naming the package, whatever the caller's + # preference. Set for this function only. + $PSNativeCommandUseErrorActionPreference = $false + $pathspecs = @(Get-PackagePaths $Package) + @(Get-PackageExcludes $Package) + $files = @(& git -C $RepoPath diff --name-only $From HEAD -- $pathspecs) + if ($LASTEXITCODE -ne 0) { + throw "git could not list the changes to $($Package.Name) since $From." + } + return $files +} + +# The files that count among a package's changed files. The repository's +# ci/release-plan-filter.ps1 decides when there is one, and otherwise every +# file counts. A path the filter returns that it was not given stops the +# plan, because a filter that rewrote a path would otherwise release a +# package on a file that never changed. +function Select-CountedFiles([string]$RepoPath, $Package, [string]$From, + [string[]]$Files) { + $filter = Join-Path $RepoPath 'ci/release-plan-filter.ps1' + if ($Files.Count -eq 0 -or (Test-Path $filter -PathType Leaf) -eq $false) { + return $Files + } + $filter = (Resolve-Path $filter).ProviderPath + $kept = @(& $filter -RepoPath $RepoPath -Package $Package -From $From ` + -Files $Files | Where-Object { $_ }) + foreach ($file in $kept) { + if ($Files -cnotcontains $file) { + throw "ci/release-plan-filter.ps1 returned '$file' for " + + "$($Package.Name), which is not one of the files it was given." + } + } + return $kept +} + # Each tox.ini that does not install every package its package depends on, # directly or through another, from this checkout, as a line naming what is # missing. Taken from a package index instead, a dependency is tested at its @@ -224,11 +307,6 @@ function Get-ReleasedVersions([string]$RepoPath, [string]$Name, # from packages.json. Packages that are not published, the examples, are never # in the plan; a change to them is tested but releases nothing. function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { - # 'git diff --quiet' exits 1 to say a package changed. That is an answer, - # not a failure, but a caller that stops on a failing native command, as - # publish-package.ps1 does, stopped there and failed the publish. Set for - # this function only. - $PSNativeCommandUseErrorActionPreference = $false if ($null -eq $Packages) { $Packages = Get-Packages $RepoPath } @@ -254,10 +332,18 @@ function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { $last = $released[-1] $previous[$package.Name] = $last $tag = "$($package.Name)/$last" - & git -C $RepoPath diff --quiet $tag HEAD -- (Get-PackagePaths $package) - if ($LASTEXITCODE -eq 1) { - $reasons[$package.Name] = "changed since $last" + $files = @(Get-ChangedFiles $RepoPath $package $tag) + $counted = @(Select-CountedFiles $RepoPath $package $tag $files) + if ($counted.Count -gt 0) { + $noun = 'files' + if ($counted.Count -eq 1) { + $noun = 'file' + } + $reasons[$package.Name] = + "changed since $last ($($counted.Count) $noun)" } + # A filter may leave an exit code that was an answer, which must not + # read as a failure to the script that called this. $global:LASTEXITCODE = 0 } diff --git a/ci/release-plan-filter.ps1 b/ci/release-plan-filter.ps1 new file mode 100644 index 00000000..dcd316aa --- /dev/null +++ b/ci/release-plan-filter.ps1 @@ -0,0 +1,39 @@ +param ( + [Parameter(Mandatory)][string]$RepoPath, + [Parameter(Mandatory)]$Package, + [Parameter(Mandatory)][string]$From, + [Parameter(Mandatory)][string[]]$Files +) +$ErrorActionPreference = "Stop" + +# Called by the release plan for each package with changed files left once +# the Ignore globs have left theirs out, and returns the files that count. +# A pyproject.toml, setup.py or setup.cfg whose change since $From is only to +# the description, keywords, urls, classifiers, readme, authors or maintainers +# does not count, which release-plan-filter.py works out. Every other file +# counts. +# +# Python is required rather than optional, because a job that counted every +# file would plan different releases from a job that could compare them. + +$manifests = 'pyproject.toml', 'setup.py', 'setup.cfg' +$compare = @($Files | Where-Object { (Split-Path -Leaf $_) -in $manifests }) +$counted = @($Files | Where-Object { (Split-Path -Leaf $_) -notin $manifests }) + +if ($compare.Count -gt 0) { + $python = Get-Command python, python3 -CommandType Application ` + -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -eq $python) { + throw "Python is needed to tell whether a change to " + + "$($compare -join ', ') is only to metadata, and none was found." + } + $PSNativeCommandUseErrorActionPreference = $false + $script = Join-Path $PSScriptRoot 'release-plan-filter.py' + $changed = @(& $python.Source -I $script $RepoPath $From @compare) + if ($LASTEXITCODE -ne 0) { + throw "release-plan-filter.py failed for $($Package.Name) since $From." + } + $counted += $changed | Where-Object { $_ } +} + +$counted diff --git a/ci/release-plan-filter.py b/ci/release-plan-filter.py new file mode 100644 index 00000000..94191c90 --- /dev/null +++ b/ci/release-plan-filter.py @@ -0,0 +1,194 @@ +"""Prints the manifest files whose change since a tag is more than metadata. + +Usage: release-plan-filter.py [ ...] + +Each file is a pyproject.toml, setup.py or setup.cfg, as a path from the +repository root. Its text at the tag and at HEAD is compared with the +description, keywords, urls, classifiers, readme, authors and maintainers +left out, so a file is printed when anything else changed, and left out when +only those did. A file that is missing on either side, or that cannot be read as what +its name says, is printed, so a release is never skipped by mistake. + +Only the standard library is used, and the same way on every Python this +repository supports, so every job of a publish works out the same plan. +""" + +import ast +import configparser +import os +import re +import subprocess +import sys + +# The setup() arguments and setup.cfg [metadata] options that are metadata. +SETUP_METADATA = frozenset(( + "description", "long_description", "long_description_content_type", + "keywords", "url", "project_urls", "download_url", "classifiers", + "author", "author_email", "maintainer", "maintainer_email")) + +# The pyproject.toml [project] keys that are metadata. +PROJECT_METADATA = frozenset(( + "description", "keywords", "urls", "classifiers", "readme", "authors", + "maintainers")) + +# A table header, such as [project.urls] or [[project.maintainers]]. +HEADER = re.compile(r"^\[\[?\s*([^\]]+?)\s*\]\]?\s*(#.*)?$") + +# The start of a key and value. The first part of a dotted key is the one +# that says what it is, as urls in urls.Homepage. +ASSIGNMENT = re.compile( + r"^(\"[^\"]*\"|'[^']*'|[A-Za-z0-9_-]+)" + r"(?:\s*\.\s*(?:\"[^\"]*\"|'[^']*'|[A-Za-z0-9_-]+))*\s*=") + + +def bytes_at(repository, revision, path): + """The file's content at the revision, or None when it is not there.""" + found = subprocess.run( + ["git", "-C", repository, "show", revision + ":" + path], + capture_output=True) + if found.returncode != 0: + return None + return found.stdout + + +class StripSetupMetadata(ast.NodeTransformer): + """Removes the metadata arguments from every call of setup().""" + + def visit_Call(self, node): + self.generic_visit(node) + name = node.func.attr if isinstance(node.func, ast.Attribute) \ + else getattr(node.func, "id", None) + if name == "setup": + node.keywords = [ + keyword for keyword in node.keywords + if keyword.arg not in SETUP_METADATA] + return node + + +def setup_py(text): + # The syntax tree leaves out comments and layout as well. + return ast.dump(StripSetupMetadata().visit(ast.parse(text))) + + +def setup_cfg(text): + parser = configparser.RawConfigParser() + parser.optionxform = str + parser.read_string(text) + return { + section: { + key: value for key, value in parser.items(section) + if section != "metadata" or key not in SETUP_METADATA} + for section in parser.sections()} + + +def toml_lines(text): + """The lines of a TOML file that are not metadata, each with the table + and key it belongs to, leaving out blank lines and comments. + + A small reader rather than tomllib, which Python 3.10 does not have. It + follows tables, keys, values that run over several lines in brackets or + in triple quotes, and comments, which is enough to place every line. + """ + kept = [] + table = "" + key = None + depth = 0 + quote = None + for line in text.splitlines(): + stripped = line.strip() + if quote is None and depth == 0: + if stripped == "" or stripped.startswith("#"): + continue + header = HEADER.match(stripped) + if header: + table = header.group(1).replace(" ", "") + key = None + if not is_metadata(table, None): + kept.append((table, None, stripped)) + continue + assignment = ASSIGNMENT.match(stripped) + if assignment is None: + # Not TOML this reader knows, so it counts unless the whole + # table is metadata. + if not is_metadata(table, "?"): + kept.append((table, "?", stripped)) + continue + key = assignment.group(1).strip("\"'") + value = stripped[assignment.end():] + else: + value = stripped + depth, quote = scan(value, depth, quote) + if not is_metadata(table, key): + kept.append((table, key, stripped)) + return kept + + +def scan(value, depth, quote): + """The bracket depth and the open triple quote after the value.""" + position = 0 + while position < len(value): + if quote is not None: + end = value.find(quote, position) + if end < 0: + return depth, quote + position = end + 3 + quote = None + continue + character = value[position] + if value.startswith('"""', position) or \ + value.startswith("'''", position): + quote = value[position:position + 3] + position += 3 + continue + if character in "\"'": + end = value.find(character, position + 1) + position = len(value) if end < 0 else end + 1 + continue + if character == "#": + break + if character in "[{": + depth += 1 + elif character in "]}": + depth -= 1 + position += 1 + return depth, quote + + +def is_metadata(table, key): + if table == "project": + return key in PROJECT_METADATA + return table in ( + "project.urls", "project.authors", "project.maintainers", + "project.readme") + + +READERS = { + "setup.py": setup_py, + "setup.cfg": setup_cfg, + "pyproject.toml": toml_lines, +} + + +def counts(repository, tag, path): + """Whether the change to the file since the tag is more than metadata.""" + reader = READERS.get(os.path.basename(path)) + before = bytes_at(repository, tag, path) + after = bytes_at(repository, "HEAD", path) + if reader is None or before is None or after is None: + return True + try: + return reader(before.decode("utf-8")) != reader(after.decode("utf-8")) + except (SyntaxError, ValueError, configparser.Error, + UnicodeDecodeError): + return True + + +def main(arguments): + repository, tag = arguments[0], arguments[1] + for path in arguments[2:]: + if counts(repository, tag, path): + print(path) + + +if __name__ == "__main__": + main(sys.argv[1:])