diff --git a/CHANGES/8097.feature b/CHANGES/8097.feature new file mode 100644 index 00000000000..f3a575f81e2 --- /dev/null +++ b/CHANGES/8097.feature @@ -0,0 +1 @@ +Added a `sha256sums` option to file repositories that publishes SHA256SUMS files alongside the content, either one at the root of the repository or one in every directory. Each file lists every file below it in the format read by `sha256sum -c`. diff --git a/pulp_file/app/migrations/0021_filepublication_sha256sums_filerepository_sha256sums.py b/pulp_file/app/migrations/0021_filepublication_sha256sums_filerepository_sha256sums.py new file mode 100644 index 00000000000..df16686b812 --- /dev/null +++ b/pulp_file/app/migrations/0021_filepublication_sha256sums_filerepository_sha256sums.py @@ -0,0 +1,48 @@ +# Generated by Django 5.2.14 on 2026-09-23 08:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("file", "0020_filerepository_last_sync_details"), + ] + + operations = [ + migrations.AddField( + model_name="filepublication", + name="sha256sums", + field=models.TextField( + choices=[ + ("disabled", "Do not generate SHA256SUMS files."), + ( + "root", + "Generate a single SHA256SUMS at the root of the repository, listing every file by its path relative to the root.", + ), + ( + "directory", + "Generate a SHA256SUMS in every directory of the repository, each listing every file below it by its path relative to that directory.", + ), + ], + default="disabled", + ), + ), + migrations.AddField( + model_name="filerepository", + name="sha256sums", + field=models.TextField( + choices=[ + ("disabled", "Do not generate SHA256SUMS files."), + ( + "root", + "Generate a single SHA256SUMS at the root of the repository, listing every file by its path relative to the root.", + ), + ( + "directory", + "Generate a SHA256SUMS in every directory of the repository, each listing every file below it by its path relative to that directory.", + ), + ], + default="disabled", + ), + ), + ] diff --git a/pulp_file/app/models.py b/pulp_file/app/models.py index 7f341a02447..7892cc537e3 100644 --- a/pulp_file/app/models.py +++ b/pulp_file/app/models.py @@ -89,7 +89,27 @@ class FileRepository(Repository, AutoAddObjPermsMixin): CONTENT_TYPES = [FileContent] REMOTE_TYPES = [FileRemote, FileGitRemote] + # Constants for the ChoiceField 'sha256sums' + SHA256SUMS_DISABLED = "disabled" + SHA256SUMS_ROOT = "root" + SHA256SUMS_DIRECTORY = "directory" + + SHA256SUMS_CHOICES = ( + (SHA256SUMS_DISABLED, "Do not generate SHA256SUMS files."), + ( + SHA256SUMS_ROOT, + "Generate a single SHA256SUMS at the root of the repository, listing every file " + "by its path relative to the root.", + ), + ( + SHA256SUMS_DIRECTORY, + "Generate a SHA256SUMS in every directory of the repository, each listing every " + "file below it by its path relative to that directory.", + ), + ) + manifest = models.TextField(default="PULP_MANIFEST", null=True) + sha256sums = models.TextField(choices=SHA256SUMS_CHOICES, default=SHA256SUMS_DISABLED) autopublish = models.BooleanField(default=False) last_sync_details = models.JSONField(default=dict) @@ -117,6 +137,7 @@ def on_new_version(self, version): if self.autopublish: tasks.publish( manifest=self.manifest, + sha256sums=self.sha256sums, repository_version_pk=version.pk, ) @@ -144,6 +165,9 @@ class FilePublication(Publication, AutoAddObjPermsMixin): TYPE = "file" manifest = models.TextField(null=True) + sha256sums = models.TextField( + choices=FileRepository.SHA256SUMS_CHOICES, default=FileRepository.SHA256SUMS_DISABLED + ) class Meta: default_related_name = "%(app_label)s_%(model_name)s" diff --git a/pulp_file/app/serializers.py b/pulp_file/app/serializers.py index eb39144d576..7d03941ca3d 100644 --- a/pulp_file/app/serializers.py +++ b/pulp_file/app/serializers.py @@ -29,6 +29,23 @@ FileRemote, FileRepository, ) +from pulp_file.manifest import Sha256Sums + + +def validate_manifest_sha256sums(manifest, sha256sums): + """ + Validate that the manifest filename does not collide with a generated SHA256SUMS file. + + Raises: + serializers.ValidationError: If the manifest would be published at a generated path. + + """ + if sha256sums != FileRepository.SHA256SUMS_DISABLED and manifest == Sha256Sums.FILENAME: + raise serializers.ValidationError( + _("The manifest cannot be named {name} while SHA256SUMS files are generated.").format( + name=Sha256Sums.FILENAME + ) + ) class FileContentSerializer(SingleArtifactContentUploadSerializer, ContentChecksumSerializer): @@ -143,13 +160,43 @@ class FileRepositorySerializer(RepositorySerializer): allow_null=True, ) + sha256sums = serializers.ChoiceField( + help_text=_( + "Whether to generate SHA256SUMS files, and where to place them. 'root' writes a " + "single file at the root of the repository, 'directory' writes one in every " + "directory. Each file lists every file below it, with paths relative to itself." + ), + choices=FileRepository.SHA256SUMS_CHOICES, + default=FileRepository.SHA256SUMS_DISABLED, + required=False, + ) + last_sync_details = serializers.JSONField( help_text=_("Details about the last sync of this repository."), read_only=True, ) + def validate(self, data): + """Validate the FileRepository data.""" + data = super().validate(data) + + validate_manifest_sha256sums( + data.get("manifest", getattr(self.instance, "manifest", None)), + data.get( + "sha256sums", + getattr(self.instance, "sha256sums", FileRepository.SHA256SUMS_DISABLED), + ), + ) + + return data + class Meta: - fields = RepositorySerializer.Meta.fields + ("autopublish", "manifest", "last_sync_details") + fields = RepositorySerializer.Meta.fields + ( + "autopublish", + "manifest", + "sha256sums", + "last_sync_details", + ) model = FileRepository @@ -218,11 +265,24 @@ class FilePublicationSerializer(PublicationSerializer): required=False, allow_null=True, ) + sha256sums = serializers.ChoiceField( + help_text=_( + "Whether to generate SHA256SUMS files, and where to place them. Defaults to the " + "setting on the repository being published." + ), + choices=FileRepository.SHA256SUMS_CHOICES, + required=False, + ) checkpoint = serializers.BooleanField(required=False) class Meta: model = FilePublication - fields = PublicationSerializer.Meta.fields + ("distributions", "manifest", "checkpoint") + fields = PublicationSerializer.Meta.fields + ( + "distributions", + "manifest", + "sha256sums", + "checkpoint", + ) class FileDistributionSerializer(DistributionSerializer): diff --git a/pulp_file/app/tasks/publishing.py b/pulp_file/app/tasks/publishing.py index 6d8f9e41966..6691296b85c 100644 --- a/pulp_file/app/tasks/publishing.py +++ b/pulp_file/app/tasks/publishing.py @@ -11,42 +11,65 @@ RepositoryVersion, ) -from pulp_file.app.models import FilePublication +from pulp_file.app.models import FilePublication, FileRepository from pulp_file.app.serializers import FilePublicationSerializer -from pulp_file.manifest import Entry, Manifest +from pulp_file.manifest import Entry, Manifest, Sha256Sums log = logging.getLogger(__name__) -def publish(manifest, repository_version_pk, checkpoint=False, **kwargs): +def publish( + manifest, + repository_version_pk, + sha256sums=FileRepository.SHA256SUMS_DISABLED, + checkpoint=False, + **kwargs, +): """ Create a Publication based on a RepositoryVersion. Args: manifest (str): Filename to use for manifest file. repository_version_pk (str): Create a publication from this repository version. + sha256sums (str): Whether to generate SHA256SUMS files, and where to place them. checkpoint (bool): Whether to create a checkpoint publication. """ repo_version = RepositoryVersion.objects.get(pk=repository_version_pk) log.info( - _("Publishing: repository={repo}, version={ver}, manifest={manifest}").format( - repo=repo_version.repository.name, ver=repo_version.number, manifest=manifest + _( + "Publishing: repository={repo}, version={ver}, manifest={manifest}, " + "sha256sums={sha256sums}" + ).format( + repo=repo_version.repository.name, + ver=repo_version.number, + manifest=manifest, + sha256sums=sha256sums, ) ) - with tempfile.TemporaryDirectory(dir="."): + with tempfile.TemporaryDirectory(dir=".") as temp_dir: with FilePublication.create( repo_version, pass_through=True, checkpoint=checkpoint ) as publication: publication.manifest = manifest + publication.sha256sums = sha256sums if manifest: manifest = Manifest(manifest) manifest.write(yield_entries_for_version(repo_version)) PublishedMetadata.create_from_file( file=File(open(manifest.relative_path, "rb")), publication=publication ) + if sha256sums != FileRepository.SHA256SUMS_DISABLED: + sums = Sha256Sums(per_directory=sha256sums == FileRepository.SHA256SUMS_DIRECTORY) + written = sums.write(yield_entries_for_version(repo_version), temp_dir) + for relative_path, local_path in written.items(): + PublishedMetadata.create_from_file( + file=File(open(local_path, "rb")), + relative_path=relative_path, + publication=publication, + ) log.info(_("Publication: {publication} created").format(publication=publication.pk)) diff --git a/pulp_file/app/viewsets.py b/pulp_file/app/viewsets.py index 0ce596ec26d..fe77e0617ba 100644 --- a/pulp_file/app/viewsets.py +++ b/pulp_file/app/viewsets.py @@ -55,6 +55,7 @@ FileRemoteSerializer, FileRepositorySerializer, FileRepositorySyncURLSerializer, + validate_manifest_sha256sums, ) @@ -574,8 +575,16 @@ def create(self, request, **kwargs): repository_version = serializer.validated_data.get("repository_version") manifest = serializer.validated_data.get("manifest") checkpoint = serializer.validated_data.get("checkpoint") + sha256sums = serializer.validated_data.get("sha256sums") + if sha256sums is None: + sha256sums = repository_version.repository.cast().sha256sums + validate_manifest_sha256sums(manifest, sha256sums) - task_kwargs = {"repository_version_pk": str(repository_version.pk), "manifest": manifest} + task_kwargs = { + "repository_version_pk": str(repository_version.pk), + "manifest": manifest, + "sha256sums": sha256sums, + } if checkpoint: task_kwargs["checkpoint"] = True diff --git a/pulp_file/docs/user/guides/publish-host.md b/pulp_file/docs/user/guides/publish-host.md index f7d0b748e8e..6f3f38c3bee 100644 --- a/pulp_file/docs/user/guides/publish-host.md +++ b/pulp_file/docs/user/guides/publish-host.md @@ -104,3 +104,33 @@ pulp file repository update --name $REPO_NAME --autopublish # This configures the distribution to be track the latest repository version for a given repository pulp file distribution update --name $DIST_NAME --repository $REPO_NAME ``` + +## Publish SHA256SUMS Files + +A repository can publish `SHA256SUMS` files alongside its content, in the format read by +`sha256sum -c`. This is off by default. Setting it to `root` publishes a single file at the root of +the repository, and `directory` publishes one in every directory. Each file lists every file below +it, with paths relative to the file itself. + +This option is not exposed by the CLI yet, so set it through the API: + +```bash +REPO_HREF=$(pulp file repository show --name $REPO_NAME | jq -r '.pulp_href') + +# Every new publication of this repository gets a SHA256SUMS in each of its directories +http PATCH :$REPO_HREF sha256sums=directory +``` + +A publication inherits the setting from the repository it is published from, and can override it: + +```bash +http POST :/pulp/api/v3/publications/file/file/ repository=$REPO_HREF sha256sums=root +``` + +Consumers verify what they downloaded by running `sha256sum -c SHA256SUMS` in the directory they +downloaded it into. Content that Pulp has not downloaded yet has no digest to publish and is left +out. + +The generated files take the `SHA256SUMS` paths for themselves, so the manifest cannot be named +`SHA256SUMS` while they are generated, and a repository already holding content at one of those +paths cannot be published. diff --git a/pulp_file/manifest.py b/pulp_file/manifest.py index 6b81811dd8c..6cf2224583e 100644 --- a/pulp_file/manifest.py +++ b/pulp_file/manifest.py @@ -1,6 +1,8 @@ +import os from collections import namedtuple from gettext import gettext as _ from re import fullmatch +from tempfile import NamedTemporaryFile Line = namedtuple("Line", ("number", "content")) @@ -143,3 +145,85 @@ def write(self, entries): line = str(entry) fp.write(line) fp.write("\n") + + +class Sha256Sums: + """ + A set of sha256sum files describing the files in a repository. + + Every file lists each file below it as ` `, the format + read by `sha256sum -c`. + + Attributes: + per_directory (bool): Write a file in every directory instead of only at the root. + + """ + + FILENAME = "SHA256SUMS" + + def __init__(self, per_directory=False): + """ + Create a new set of sha256sum files. + + Args: + per_directory (bool): Write a file in every directory instead of only at the root. + + """ + self.per_directory = per_directory + + def write(self, entries, dest_dir): + """ + Write the files. + + Entries without a digest are skipped, since on-demand content may not have one. + + Args: + entries (iterable): The entries to be listed. + dest_dir (str): An existing directory to write the files into. + + Returns: + dict: The relative path to publish each file at, mapped to its path on disk. + + """ + written = {} + open_files = {} + try: + # Sorting keeps each directory's entries contiguous, so its file can be closed on the way out. + for entry in sorted(entries, key=lambda entry: entry.relative_path): + if not entry.digest: + continue + directories = list(self._directories(entry.relative_path)) + for directory in set(open_files).difference(directories): + open_files.pop(directory).close() + for directory in directories: + if directory not in open_files: + fp = NamedTemporaryFile(mode="w", dir=dest_dir, delete=False) + open_files[directory] = fp + written[os.path.join(directory, self.FILENAME)] = fp.name + if directory: + name = entry.relative_path[len(directory) + 1 :] + else: + name = entry.relative_path + open_files[directory].write(f"{entry.digest} {name}\n") + finally: + for fp in open_files.values(): + fp.close() + + return written + + def _directories(self, relative_path): + """ + Yield the directories whose file should list `relative_path`, outermost first. + + Args: + relative_path (str): A relative path. + + Yields: + str: A directory, relative to the root of the repository. + + """ + yield "" + if self.per_directory: + parts = relative_path.split("/")[:-1] + for depth in range(1, len(parts) + 1): + yield "/".join(parts[:depth]) diff --git a/pulp_file/tests/functional/api/test_sha256sums.py b/pulp_file/tests/functional/api/test_sha256sums.py new file mode 100644 index 00000000000..3a9b1f878e1 --- /dev/null +++ b/pulp_file/tests/functional/api/test_sha256sums.py @@ -0,0 +1,116 @@ +"""Tests that publish SHA256SUMS files alongside file plugin content.""" + +import pytest +import requests + +RELATIVE_PATHS = ["release.iso", "docs/intro.md", "docs/api/reference.md"] + + +@pytest.fixture +def sha256sums_repository_factory( + file_bindings, file_repository_factory, file_content_unit_with_name_factory, monitor_task +): + """A factory to generate a repository holding a file at each of RELATIVE_PATHS.""" + + def _sha256sums_repository_factory(**body): + repository = file_repository_factory(**body) + units = [file_content_unit_with_name_factory(path) for path in RELATIVE_PATHS] + monitor_task( + file_bindings.RepositoriesFileApi.modify( + repository.pulp_href, {"add_content_units": [unit.pulp_href for unit in units]} + ).task + ) + return repository, {unit.relative_path: unit.sha256 for unit in units} + + return _sha256sums_repository_factory + + +@pytest.fixture +def sha256sums_distribution_factory( + sha256sums_repository_factory, file_distribution_factory, distribution_base_url +): + """A factory to generate an autopublished distribution of such a repository.""" + + def _sha256sums_distribution_factory(**body): + repository, digests = sha256sums_repository_factory(autopublish=True, **body) + distribution = file_distribution_factory(repository=repository.pulp_href) + return distribution_base_url(distribution.base_url), digests + + return _sha256sums_distribution_factory + + +def get_sha256sums(base_url, directory=""): + """Return the entries of the SHA256SUMS in `directory`, or None if it is not served.""" + response = requests.get(f"{base_url}{directory}SHA256SUMS") + if response.status_code == 404: + return None + response.raise_for_status() + return {tuple(line.split(" ")) for line in response.text.splitlines()} + + +@pytest.mark.parallel +def test_disabled_by_default(sha256sums_distribution_factory): + base_url, _ = sha256sums_distribution_factory() + + assert get_sha256sums(base_url) is None + + +@pytest.mark.parallel +def test_root(sha256sums_distribution_factory): + base_url, digests = sha256sums_distribution_factory(sha256sums="root") + + assert get_sha256sums(base_url) == {(digest, path) for path, digest in digests.items()} + assert get_sha256sums(base_url, "docs/") is None + assert get_sha256sums(base_url, "docs/api/") is None + + +@pytest.mark.parallel +def test_directory(sha256sums_distribution_factory): + base_url, digests = sha256sums_distribution_factory(sha256sums="directory") + + assert get_sha256sums(base_url) == {(digest, path) for path, digest in digests.items()} + assert get_sha256sums(base_url, "docs/") == { + (digests["docs/intro.md"], "intro.md"), + (digests["docs/api/reference.md"], "api/reference.md"), + } + assert get_sha256sums(base_url, "docs/api/") == { + (digests["docs/api/reference.md"], "reference.md") + } + + +@pytest.mark.parallel +def test_publication_defaults_to_repository( + sha256sums_repository_factory, file_publication_factory +): + repository, _ = sha256sums_repository_factory(sha256sums="directory") + + publication = file_publication_factory(repository=repository.pulp_href, sha256sums="root") + assert publication.sha256sums == "root" + + publication = file_publication_factory(repository=repository.pulp_href) + assert publication.sha256sums == "directory" + + +@pytest.mark.parallel +def test_repository_manifest_cannot_collide(file_bindings, file_repository_factory): + with pytest.raises(file_bindings.ApiException) as exc: + file_repository_factory(manifest="SHA256SUMS", sha256sums="root") + assert exc.value.status == 400 + + repository = file_repository_factory(sha256sums="root") + with pytest.raises(file_bindings.ApiException) as exc: + file_bindings.RepositoriesFileApi.partial_update( + repository.pulp_href, {"manifest": "SHA256SUMS"} + ) + assert exc.value.status == 400 + + +@pytest.mark.parallel +def test_publication_manifest_cannot_collide(file_bindings, file_repository_factory): + repository = file_repository_factory(sha256sums="root") + + with pytest.raises(file_bindings.ApiException) as exc: + file_bindings.PublicationsFileApi.create( + {"repository": repository.pulp_href, "manifest": "SHA256SUMS"} + ) + assert exc.value.status == 400 diff --git a/pulp_file/tests/unit/test_sha256sums.py b/pulp_file/tests/unit/test_sha256sums.py new file mode 100644 index 00000000000..aa67ba58a82 --- /dev/null +++ b/pulp_file/tests/unit/test_sha256sums.py @@ -0,0 +1,86 @@ +import pytest + +from pulp_file import manifest as manifest_module +from pulp_file.manifest import Entry, Sha256Sums + +ENTRIES = [ + Entry(relative_path="release.iso", digest="a" * 64, size=1), + Entry(relative_path="docs/intro.md", digest="b" * 64, size=2), + Entry(relative_path="docs/api/reference.md", digest="c" * 64, size=3), +] + + +def read_written(written): + return {path: open(local_path).read() for path, local_path in written.items()} + + +def test_root(tmp_path): + written = Sha256Sums().write(ENTRIES, str(tmp_path)) + + assert read_written(written) == { + "SHA256SUMS": ( + f"{'c' * 64} docs/api/reference.md\n" + f"{'b' * 64} docs/intro.md\n" + f"{'a' * 64} release.iso\n" + ) + } + + +def test_per_directory(tmp_path): + written = Sha256Sums(per_directory=True).write(ENTRIES, str(tmp_path)) + + assert read_written(written) == { + "SHA256SUMS": ( + f"{'c' * 64} docs/api/reference.md\n" + f"{'b' * 64} docs/intro.md\n" + f"{'a' * 64} release.iso\n" + ), + "docs/SHA256SUMS": f"{'c' * 64} api/reference.md\n{'b' * 64} intro.md\n", + "docs/api/SHA256SUMS": f"{'c' * 64} reference.md\n", + } + + +@pytest.mark.parametrize("per_directory", [False, True]) +def test_no_entries(tmp_path, per_directory): + assert Sha256Sums(per_directory=per_directory).write([], str(tmp_path)) == {} + + +def test_entries_without_digest_are_skipped(tmp_path): + entries = ENTRIES + [Entry(relative_path="docs/ondemand.md", digest=None, size=4)] + + written = Sha256Sums(per_directory=True).write(entries, str(tmp_path)) + + assert "ondemand.md" not in read_written(written)["docs/SHA256SUMS"] + + +def test_open_files_are_bound_by_depth(tmp_path, monkeypatch): + entries = [ + Entry(relative_path=f"dir{n}/sub/file.txt", digest=f"{n:064}", size=1) for n in range(100) + ] + open_files = 0 + high_water_mark = 0 + named_temporary_file = manifest_module.NamedTemporaryFile + + def counting_named_temporary_file(*args, **kwargs): + nonlocal open_files, high_water_mark + fp = named_temporary_file(*args, **kwargs) + close = fp.close + open_files += 1 + high_water_mark = max(high_water_mark, open_files) + + def counting_close(): + nonlocal open_files + if not fp.closed: + open_files -= 1 + close() + + fp.close = counting_close + return fp + + monkeypatch.setattr(manifest_module, "NamedTemporaryFile", counting_named_temporary_file) + + written = Sha256Sums(per_directory=True).write(entries, str(tmp_path)) + + assert len(written) == 201 + assert open_files == 0 + assert high_water_mark == 3