Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions .ci/check_package.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
"""Build, audit and test Python distributions in isolation from the source checkout."""
import argparse
import email.parser
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tarfile
import tempfile
import tomllib
import zipfile

ROOT = Path(__file__).resolve().parents[1]


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
output = args.output.resolve()
dist, metadata = output / "dist", output / "manifest"
if output.exists() and any(output.iterdir()):
raise SystemExit("distribution destination must be empty")
dist.mkdir(parents=True)
metadata.mkdir()
Path("/var/tmp").mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="peru-python-package-", dir="/var/tmp") as directory:
task = Path(directory)
env = {**os.environ, "TMPDIR": directory, "TEMP": directory, "TMP": directory,
"PYTHONDONTWRITEBYTECODE": "1", "PIP_DISABLE_PIP_VERSION_CHECK": "1"}

def run(*command, cwd=task, extra=None):
subprocess.run(command, cwd=cwd, env={**env, **(extra or {})}, check=True)

source = task / "source"
shutil.copytree(ROOT, source, ignore=shutil.ignore_patterns(".git", "__pycache__", "*.egg-info", "build", "dist"))
project = tomllib.loads((source / "pyproject.toml").read_text())["project"]
run(sys.executable, "-m", "build", str(source), "--outdir", str(dist))
wheel, = dist.glob("*.whl")
sdist, = dist.glob("*.tar.gz")
with zipfile.ZipFile(wheel) as archive:
names = archive.namelist()
if len(names) != len(set(names)):
raise SystemExit("duplicate wheel entries")
metadata_name, = [name for name in names if name.endswith(".dist-info/METADATA")]
info_root = metadata_name.rsplit("/", 1)[0] + "/"
for name in names:
if ".." in Path(name).parts or not name.startswith(("peru_sdk/", info_root)):
raise SystemExit(f"unexpected wheel file: {name}")
if name.endswith((".go", ".java", ".so", ".pyd", ".dll", ".whl", ".jar")):
raise SystemExit(f"foreign implementation or binary in wheel: {name}")
fields = email.parser.BytesParser().parsebytes(archive.read(metadata_name))
if fields["Name"] != project["name"] or fields["Version"] != project["version"] or fields.get_all("Requires-Dist"):
raise SystemExit("unexpected package identity, version or runtime dependencies")
extracted = task / "sdist"
with tarfile.open(sdist) as archive:
for member in archive.getmembers():
path = Path(member.name)
if path.is_absolute() or ".." in path.parts or not (member.isfile() or member.isdir()):
raise SystemExit("unsafe source archive entry")
target = extracted / path
if member.isdir():
target.mkdir(parents=True, exist_ok=True)
else:
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(archive.extractfile(member).read())
package_root, = extracted.iterdir()
if not (package_root / "tests/data/conformance.json").is_file():
raise SystemExit("source distribution is missing its independent test corpus")
for path in package_root.rglob("*"):
if path.suffix in (".go", ".java", ".jar", ".whl") or path.name in ("go.mod", "sdk-go"):
raise SystemExit("foreign SDK in source distribution")
site = task / "installed"
run(sys.executable, "-m", "pip", "install", "--no-deps", "--no-cache-dir", "--only-binary=:all:",
"--target", str(site), str(wheel))
extra = {"PYTHONPATH": str(site)}
run(sys.executable, "-c", "import peru_sdk; from pathlib import Path; assert Path(peru_sdk.__file__).is_relative_to(Path('installed').resolve())", extra=extra)
run(sys.executable, "-m", "unittest", "discover", "-s", str(package_root / "tests"), "-v", extra=extra)
run(sys.executable, "-m", "twine", "check", "--strict", str(wheel), str(sdist))
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
release = {"project": project["name"], "version": project["version"], "commit": commit,
"upstreams": json.loads((ROOT / "upstream.lock.json").read_text())}
(metadata / "release.json").write_text(json.dumps(release, indent=2) + "\n")
sums = []
for path in sorted(output.rglob("*")):
if path.is_file():
sums.append(f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.relative_to(output).as_posix()}")
(metadata / "SHA256SUMS").write_text("\n".join(sums) + "\n")
print("Verified isolated Python distributions:", output)


if __name__ == "__main__":
main()
18 changes: 18 additions & 0 deletions .ci/check_release.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
"""Require a release tag to match the Python project version."""
import os
from pathlib import Path
import tomllib

ROOT = Path(__file__).resolve().parents[1]


def main():
version = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"]["version"]
expected = "refs/tags/v" + version
if os.environ.get("GITHUB_REF") != expected:
raise SystemExit(f"Publishing requires tag v{version}; project/tag mismatch")
print("Release tag matches project version:", version)


if __name__ == "__main__":
main()
2 changes: 2 additions & 0 deletions .ci/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
build>=1.2,<2
twine>=6,<8
85 changes: 85 additions & 0 deletions .ci/sync_upstream.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
"""Explicit, integrity-checked development synchronization; never used at runtime."""
import argparse
import hashlib
import json
from pathlib import Path
import re
import subprocess
import sys
import tempfile
from urllib.request import urlopen

ROOT = Path(__file__).resolve().parents[1]


def fetch(source: dict, path: str, digest: str) -> bytes:
repository, revision = source["repository"], source["revision"]
if not re.fullmatch(r"DataScienceResearchPeru/peru-sdk-(models|conformance)", repository):
raise ValueError("unexpected upstream repository")
if not re.fullmatch(r"[0-9a-f]{40}", revision) or not re.fullmatch(r"[0-9a-f]{64}", digest):
raise ValueError("upstreams require full commits and SHA-256 hashes")
relative = Path(path)
if relative.is_absolute() or ".." in relative.parts:
raise ValueError("invalid upstream path")
with urlopen(f"https://raw.githubusercontent.com/{repository}/{revision}/{path}", timeout=30) as response:
data = response.read(2 * 1024 * 1024 + 1)
if len(data) > 2 * 1024 * 1024 or hashlib.sha256(data).hexdigest() != digest:
raise ValueError(f"upstream integrity mismatch: {path}")
return data


def synchronize(check: bool):
lock = json.loads((ROOT / "upstream.lock.json").read_text(encoding="utf-8"))
if lock["format_version"] != 1:
raise ValueError("unsupported lock format")
case_path = Path(lock["cases_file"])
if case_path.is_absolute() or ".." in case_path.parts:
raise ValueError("invalid corpus destination")
source = lock["conformance"]
corpus = fetch(source, source["path"], source["sha256"])
outputs = {ROOT / case_path: corpus}
Path("/var/tmp").mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix="peru-sdk-upstream-", dir="/var/tmp") as directory:
task = Path(directory)
if "generator" in lock:
language = lock["language"]
if language not in ("go", "python"):
raise ValueError("unsupported language")
generator = lock["generator"]
for name, digest in generator["files"].items():
data = fetch(generator, name, digest)
target = task / "models" / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(data)
destination = task / "output"
subprocess.run([sys.executable, str(task / "models/tools/generate.py"),
"--language", language, "--output", str(destination)], check=True)
root = ROOT if language == "go" else ROOT / "src/peru_sdk"
for path in destination.rglob("*"):
if path.is_file():
outputs[root / path.relative_to(destination)] = path.read_bytes()
differences = []
for path, content in outputs.items():
if check:
if not path.exists() or path.read_bytes() != content:
differences.append(str(path.relative_to(ROOT)))
else:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
if differences:
raise ValueError("regeneration required: " + ", ".join(differences))
print("Pinned upstreams and local outputs match." if check else "Updated from pinned upstreams.")


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--check", action="store_true")
args = parser.parse_args()
try:
synchronize(args.check)
except (ValueError, KeyError, OSError) as error:
parser.exit(1, str(error) + "\n")


if __name__ == "__main__":
main()
32 changes: 32 additions & 0 deletions .ci/test_release.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import importlib.util
import os
from pathlib import Path
import tomllib
import unittest
from unittest.mock import patch

SPEC = importlib.util.spec_from_file_location("release_check", Path(__file__).with_name("check_release.py"))
release = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(release)
VERSION = tomllib.loads((release.ROOT / "pyproject.toml").read_text())["project"]["version"]


class ReleaseGuardTests(unittest.TestCase):
def test_matching_version_tag(self):
with patch.dict(os.environ, {"GITHUB_REF": "refs/tags/v" + VERSION}):
release.main()

def test_main_is_not_a_release(self):
with patch.dict(os.environ, {"GITHUB_REF": "refs/heads/main"}):
with self.assertRaises(SystemExit):
release.main()

def test_tag_for_another_version_is_rejected(self):
other_version = "0.0.0" if VERSION != "0.0.0" else "0.0.1"
with patch.dict(os.environ, {"GITHUB_REF": "refs/tags/v" + other_version}):
with self.assertRaises(SystemExit):
release.main()


if __name__ == "__main__":
unittest.main()
42 changes: 42 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: CI
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
env:
PYTHONDONTWRITEBYTECODE: "1"
jobs:
upstream:
runs-on: ubuntu-latest
timeout-minutes: 10
env:
TMPDIR: /var/tmp
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0
with:
python-version: "3.14"
- run: python .ci/sync_upstream.py --check
native:
name: Python ${{ matrix.python }} / ${{ matrix.os }}
runs-on: ${{ matrix.os }}
timeout-minutes: 10
strategy:
fail-fast: false
max-parallel: 2
matrix:
python: ["3.11", "3.14"]
os: [ubuntu-latest, windows-latest]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0
with:
python-version: ${{ matrix.python }}
- run: python -m pip install .
- run: python -m unittest discover -s tests -v
68 changes: 68 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
name: Release
on:
workflow_dispatch:
inputs:
publish:
description: Publish the verified version
type: boolean
default: false
pull_request:
push:
tags: ["v*"]
permissions:
contents: read
concurrency:
group: release-${{ github.ref }}
cancel-in-progress: false
jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 15
env:
TMPDIR: /var/tmp
PYTHONDONTWRITEBYTECODE: "1"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0
with:
python-version: "3.14"
- run: python -m pip install -r .ci/requirements.txt
- run: python .ci/check_release.py
if: github.event_name == 'push' || inputs.publish
- run: python .ci/test_release.py
- run: python .ci/sync_upstream.py --check
- name: Build and test isolated distributions
run: |
stage=$(mktemp -d /var/tmp/peru-python-release.XXXXXXXX)
echo "SDK_DIST=$stage" >> "$GITHUB_ENV"
python .ci/check_package.py --output "$stage"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
with:
name: python-release-${{ github.sha }}
path: ${{ env.SDK_DIST }}
retention-days: 30
publish:
if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish)
needs: build
runs-on: ubuntu-latest
timeout-minutes: 10
environment: pypi
permissions:
id-token: write
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333
with:
name: python-release-${{ github.sha }}
path: release
- name: Verify artifacts
run: sha256sum -c manifest/SHA256SUMS
working-directory: release
- uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086
with:
packages-dir: release/dist/
11 changes: 11 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
__pycache__/
*.py[cod]
*.egg-info/
build/
dist/
.venv/
.env
*.pfx
*.p12
*.pem
*.key
2 changes: 2 additions & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
include LICENSE README.md upstream.lock.json
recursive-include tests *.py *.json
Loading
Loading