From 6ece3ce0e846c0dd6594ff8c6111aced7bf6ea9c Mon Sep 17 00:00:00 2001 From: Aliaksei Klimau Date: Wed, 23 Sep 2026 13:10:31 +0200 Subject: [PATCH 1/2] Add optional SHA256SUMS generation to file publications File repositories gain a sha256sums option that publishes SHA256SUMS files alongside the content, in the format read by sha256sum -c. It is disabled by default. 'root' writes a single file at the root of the repository, and 'directory' writes one in every directory; each file lists every file below it by its path relative to itself, so a nested directory can be verified on its own and a deleted one is still detected by its parent. Publications accept the same option and fall back to the repository's setting when it is omitted, so autopublish and explicit publishes behave alike. On-demand content with no known digest is skipped. Closes #8097 Co-Authored-By: Claude Opus 5 --- CHANGES/8097.feature | 1 + ...on_sha256sums_filerepository_sha256sums.py | 48 ++++++++++ pulp_file/app/models.py | 24 +++++ pulp_file/app/serializers.py | 33 ++++++- pulp_file/app/tasks/publishing.py | 35 +++++-- pulp_file/app/viewsets.py | 9 +- pulp_file/docs/user/guides/publish-host.md | 26 ++++++ pulp_file/manifest.py | 80 ++++++++++++++++ .../tests/functional/api/test_sha256sums.py | 91 +++++++++++++++++++ pulp_file/tests/unit/test_sha256sums.py | 52 +++++++++++ 10 files changed, 390 insertions(+), 9 deletions(-) create mode 100644 CHANGES/8097.feature create mode 100644 pulp_file/app/migrations/0021_filepublication_sha256sums_filerepository_sha256sums.py create mode 100644 pulp_file/tests/functional/api/test_sha256sums.py create mode 100644 pulp_file/tests/unit/test_sha256sums.py diff --git a/CHANGES/8097.feature b/CHANGES/8097.feature new file mode 100644 index 00000000000..f3a575f81e2 --- /dev/null +++ b/CHANGES/8097.feature @@ -0,0 +1 @@ +Added a `sha256sums` option to file repositories that publishes SHA256SUMS files alongside the content, either one at the root of the repository or one in every directory. Each file lists every file below it in the format read by `sha256sum -c`. diff --git a/pulp_file/app/migrations/0021_filepublication_sha256sums_filerepository_sha256sums.py b/pulp_file/app/migrations/0021_filepublication_sha256sums_filerepository_sha256sums.py new file mode 100644 index 00000000000..df16686b812 --- /dev/null +++ b/pulp_file/app/migrations/0021_filepublication_sha256sums_filerepository_sha256sums.py @@ -0,0 +1,48 @@ +# Generated by Django 5.2.14 on 2026-09-23 08:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("file", "0020_filerepository_last_sync_details"), + ] + + operations = [ + migrations.AddField( + model_name="filepublication", + name="sha256sums", + field=models.TextField( + choices=[ + ("disabled", "Do not generate SHA256SUMS files."), + ( + "root", + "Generate a single SHA256SUMS at the root of the repository, listing every file by its path relative to the root.", + ), + ( + "directory", + "Generate a SHA256SUMS in every directory of the repository, each listing every file below it by its path relative to that directory.", + ), + ], + default="disabled", + ), + ), + migrations.AddField( + model_name="filerepository", + name="sha256sums", + field=models.TextField( + choices=[ + ("disabled", "Do not generate SHA256SUMS files."), + ( + "root", + "Generate a single SHA256SUMS at the root of the repository, listing every file by its path relative to the root.", + ), + ( + "directory", + "Generate a SHA256SUMS in every directory of the repository, each listing every file below it by its path relative to that directory.", + ), + ], + default="disabled", + ), + ), + ] diff --git a/pulp_file/app/models.py b/pulp_file/app/models.py index 7f341a02447..7892cc537e3 100644 --- a/pulp_file/app/models.py +++ b/pulp_file/app/models.py @@ -89,7 +89,27 @@ class FileRepository(Repository, AutoAddObjPermsMixin): CONTENT_TYPES = [FileContent] REMOTE_TYPES = [FileRemote, FileGitRemote] + # Constants for the ChoiceField 'sha256sums' + SHA256SUMS_DISABLED = "disabled" + SHA256SUMS_ROOT = "root" + SHA256SUMS_DIRECTORY = "directory" + + SHA256SUMS_CHOICES = ( + (SHA256SUMS_DISABLED, "Do not generate SHA256SUMS files."), + ( + SHA256SUMS_ROOT, + "Generate a single SHA256SUMS at the root of the repository, listing every file " + "by its path relative to the root.", + ), + ( + SHA256SUMS_DIRECTORY, + "Generate a SHA256SUMS in every directory of the repository, each listing every " + "file below it by its path relative to that directory.", + ), + ) + manifest = models.TextField(default="PULP_MANIFEST", null=True) + sha256sums = models.TextField(choices=SHA256SUMS_CHOICES, default=SHA256SUMS_DISABLED) autopublish = models.BooleanField(default=False) last_sync_details = models.JSONField(default=dict) @@ -117,6 +137,7 @@ def on_new_version(self, version): if self.autopublish: tasks.publish( manifest=self.manifest, + sha256sums=self.sha256sums, repository_version_pk=version.pk, ) @@ -144,6 +165,9 @@ class FilePublication(Publication, AutoAddObjPermsMixin): TYPE = "file" manifest = models.TextField(null=True) + sha256sums = models.TextField( + choices=FileRepository.SHA256SUMS_CHOICES, default=FileRepository.SHA256SUMS_DISABLED + ) class Meta: default_related_name = "%(app_label)s_%(model_name)s" diff --git a/pulp_file/app/serializers.py b/pulp_file/app/serializers.py index eb39144d576..0b74aa4ff7d 100644 --- a/pulp_file/app/serializers.py +++ b/pulp_file/app/serializers.py @@ -143,13 +143,29 @@ class FileRepositorySerializer(RepositorySerializer): allow_null=True, ) + sha256sums = serializers.ChoiceField( + help_text=_( + "Whether to generate SHA256SUMS files, and where to place them. 'root' writes a " + "single file at the root of the repository, 'directory' writes one in every " + "directory. Each file lists every file below it, with paths relative to itself." + ), + choices=FileRepository.SHA256SUMS_CHOICES, + default=FileRepository.SHA256SUMS_DISABLED, + required=False, + ) + last_sync_details = serializers.JSONField( help_text=_("Details about the last sync of this repository."), read_only=True, ) class Meta: - fields = RepositorySerializer.Meta.fields + ("autopublish", "manifest", "last_sync_details") + fields = RepositorySerializer.Meta.fields + ( + "autopublish", + "manifest", + "sha256sums", + "last_sync_details", + ) model = FileRepository @@ -218,11 +234,24 @@ class FilePublicationSerializer(PublicationSerializer): required=False, allow_null=True, ) + sha256sums = serializers.ChoiceField( + help_text=_( + "Whether to generate SHA256SUMS files, and where to place them. Defaults to the " + "setting on the repository being published." + ), + choices=FileRepository.SHA256SUMS_CHOICES, + required=False, + ) checkpoint = serializers.BooleanField(required=False) class Meta: model = FilePublication - fields = PublicationSerializer.Meta.fields + ("distributions", "manifest", "checkpoint") + fields = PublicationSerializer.Meta.fields + ( + "distributions", + "manifest", + "sha256sums", + "checkpoint", + ) class FileDistributionSerializer(DistributionSerializer): diff --git a/pulp_file/app/tasks/publishing.py b/pulp_file/app/tasks/publishing.py index 6d8f9e41966..6691296b85c 100644 --- a/pulp_file/app/tasks/publishing.py +++ b/pulp_file/app/tasks/publishing.py @@ -11,42 +11,65 @@ RepositoryVersion, ) -from pulp_file.app.models import FilePublication +from pulp_file.app.models import FilePublication, FileRepository from pulp_file.app.serializers import FilePublicationSerializer -from pulp_file.manifest import Entry, Manifest +from pulp_file.manifest import Entry, Manifest, Sha256Sums log = logging.getLogger(__name__) -def publish(manifest, repository_version_pk, checkpoint=False, **kwargs): +def publish( + manifest, + repository_version_pk, + sha256sums=FileRepository.SHA256SUMS_DISABLED, + checkpoint=False, + **kwargs, +): """ Create a Publication based on a RepositoryVersion. Args: manifest (str): Filename to use for manifest file. repository_version_pk (str): Create a publication from this repository version. + sha256sums (str): Whether to generate SHA256SUMS files, and where to place them. checkpoint (bool): Whether to create a checkpoint publication. """ repo_version = RepositoryVersion.objects.get(pk=repository_version_pk) log.info( - _("Publishing: repository={repo}, version={ver}, manifest={manifest}").format( - repo=repo_version.repository.name, ver=repo_version.number, manifest=manifest + _( + "Publishing: repository={repo}, version={ver}, manifest={manifest}, " + "sha256sums={sha256sums}" + ).format( + repo=repo_version.repository.name, + ver=repo_version.number, + manifest=manifest, + sha256sums=sha256sums, ) ) - with tempfile.TemporaryDirectory(dir="."): + with tempfile.TemporaryDirectory(dir=".") as temp_dir: with FilePublication.create( repo_version, pass_through=True, checkpoint=checkpoint ) as publication: publication.manifest = manifest + publication.sha256sums = sha256sums if manifest: manifest = Manifest(manifest) manifest.write(yield_entries_for_version(repo_version)) PublishedMetadata.create_from_file( file=File(open(manifest.relative_path, "rb")), publication=publication ) + if sha256sums != FileRepository.SHA256SUMS_DISABLED: + sums = Sha256Sums(per_directory=sha256sums == FileRepository.SHA256SUMS_DIRECTORY) + written = sums.write(yield_entries_for_version(repo_version), temp_dir) + for relative_path, local_path in written.items(): + PublishedMetadata.create_from_file( + file=File(open(local_path, "rb")), + relative_path=relative_path, + publication=publication, + ) log.info(_("Publication: {publication} created").format(publication=publication.pk)) diff --git a/pulp_file/app/viewsets.py b/pulp_file/app/viewsets.py index 0ce596ec26d..41c7180fab5 100644 --- a/pulp_file/app/viewsets.py +++ b/pulp_file/app/viewsets.py @@ -574,8 +574,15 @@ def create(self, request, **kwargs): repository_version = serializer.validated_data.get("repository_version") manifest = serializer.validated_data.get("manifest") checkpoint = serializer.validated_data.get("checkpoint") + sha256sums = serializer.validated_data.get("sha256sums") + if sha256sums is None: + sha256sums = repository_version.repository.cast().sha256sums - task_kwargs = {"repository_version_pk": str(repository_version.pk), "manifest": manifest} + task_kwargs = { + "repository_version_pk": str(repository_version.pk), + "manifest": manifest, + "sha256sums": sha256sums, + } if checkpoint: task_kwargs["checkpoint"] = True diff --git a/pulp_file/docs/user/guides/publish-host.md b/pulp_file/docs/user/guides/publish-host.md index f7d0b748e8e..134a69ffe5b 100644 --- a/pulp_file/docs/user/guides/publish-host.md +++ b/pulp_file/docs/user/guides/publish-host.md @@ -104,3 +104,29 @@ pulp file repository update --name $REPO_NAME --autopublish # This configures the distribution to be track the latest repository version for a given repository pulp file distribution update --name $DIST_NAME --repository $REPO_NAME ``` + +## Publish SHA256SUMS Files + +A repository can publish `SHA256SUMS` files alongside its content, in the format read by +`sha256sum -c`. This is off by default. Setting it to `root` publishes a single file at the root of +the repository, and `directory` publishes one in every directory. Each file lists every file below +it, with paths relative to the file itself. + +This option is not exposed by the CLI yet, so set it through the API: + +```bash +REPO_HREF=$(pulp file repository show --name $REPO_NAME | jq -r '.pulp_href') + +# Every new publication of this repository gets a SHA256SUMS in each of its directories +http PATCH :$REPO_HREF sha256sums=directory +``` + +A publication inherits the setting from the repository it is published from, and can override it: + +```bash +http POST :/pulp/api/v3/publications/file/file/ repository=$REPO_HREF sha256sums=root +``` + +Consumers verify what they downloaded by running `sha256sum -c SHA256SUMS` in the directory they +downloaded it into. Content that Pulp has not downloaded yet has no digest to publish and is left +out. diff --git a/pulp_file/manifest.py b/pulp_file/manifest.py index 6b81811dd8c..156e0bbd3b8 100644 --- a/pulp_file/manifest.py +++ b/pulp_file/manifest.py @@ -1,6 +1,8 @@ +import os from collections import namedtuple from gettext import gettext as _ from re import fullmatch +from tempfile import NamedTemporaryFile Line = namedtuple("Line", ("number", "content")) @@ -143,3 +145,81 @@ def write(self, entries): line = str(entry) fp.write(line) fp.write("\n") + + +class Sha256Sums: + """ + A set of sha256sum files describing the files in a repository. + + Every file lists each file below it as ` `, the format + read by `sha256sum -c`. + + Attributes: + per_directory (bool): Write a file in every directory instead of only at the root. + + """ + + FILENAME = "SHA256SUMS" + + def __init__(self, per_directory=False): + """ + Create a new set of sha256sum files. + + Args: + per_directory (bool): Write a file in every directory instead of only at the root. + + """ + self.per_directory = per_directory + + def write(self, entries, dest_dir): + """ + Write the files. + + Entries without a digest are skipped, since on-demand content may not have one. + + Args: + entries (iterable): The entries to be listed. + dest_dir (str): An existing directory to write the files into. + + Returns: + dict: The relative path to publish each file at, mapped to its path on disk. + + """ + open_files = {} + try: + for entry in entries: + if not entry.digest: + continue + for directory in self._directories(entry.relative_path): + sums_path = os.path.join(directory, self.FILENAME) + if sums_path not in open_files: + open_files[sums_path] = NamedTemporaryFile( + mode="w", dir=dest_dir, delete=False + ) + if directory: + name = entry.relative_path[len(directory) + 1 :] + else: + name = entry.relative_path + open_files[sums_path].write(f"{entry.digest} {name}\n") + finally: + for fp in open_files.values(): + fp.close() + + return {sums_path: fp.name for sums_path, fp in open_files.items()} + + def _directories(self, relative_path): + """ + Yield the directories whose file should list `relative_path`, outermost first. + + Args: + relative_path (str): A relative path. + + Yields: + str: A directory, relative to the root of the repository. + + """ + yield "" + if self.per_directory: + parts = relative_path.split("/")[:-1] + for depth in range(1, len(parts) + 1): + yield "/".join(parts[:depth]) diff --git a/pulp_file/tests/functional/api/test_sha256sums.py b/pulp_file/tests/functional/api/test_sha256sums.py new file mode 100644 index 00000000000..816b734fff1 --- /dev/null +++ b/pulp_file/tests/functional/api/test_sha256sums.py @@ -0,0 +1,91 @@ +"""Tests that publish SHA256SUMS files alongside file plugin content.""" + +import pytest +import requests + +RELATIVE_PATHS = ["release.iso", "docs/intro.md", "docs/api/reference.md"] + + +@pytest.fixture +def sha256sums_repository_factory( + file_bindings, file_repository_factory, file_content_unit_with_name_factory, monitor_task +): + """A factory to generate a repository holding a file at each of RELATIVE_PATHS.""" + + def _sha256sums_repository_factory(**body): + repository = file_repository_factory(**body) + units = [file_content_unit_with_name_factory(path) for path in RELATIVE_PATHS] + monitor_task( + file_bindings.RepositoriesFileApi.modify( + repository.pulp_href, {"add_content_units": [unit.pulp_href for unit in units]} + ).task + ) + return repository, {unit.relative_path: unit.sha256 for unit in units} + + return _sha256sums_repository_factory + + +@pytest.fixture +def sha256sums_distribution_factory( + sha256sums_repository_factory, file_distribution_factory, distribution_base_url +): + """A factory to generate an autopublished distribution of such a repository.""" + + def _sha256sums_distribution_factory(**body): + repository, digests = sha256sums_repository_factory(autopublish=True, **body) + distribution = file_distribution_factory(repository=repository.pulp_href) + return distribution_base_url(distribution.base_url), digests + + return _sha256sums_distribution_factory + + +def get_sha256sums(base_url, directory=""): + """Return the entries of the SHA256SUMS in `directory`, or None if it is not served.""" + response = requests.get(f"{base_url}{directory}SHA256SUMS") + if response.status_code == 404: + return None + response.raise_for_status() + return {tuple(line.split(" ")) for line in response.text.splitlines()} + + +@pytest.mark.parallel +def test_disabled_by_default(sha256sums_distribution_factory): + base_url, _ = sha256sums_distribution_factory() + + assert get_sha256sums(base_url) is None + + +@pytest.mark.parallel +def test_root(sha256sums_distribution_factory): + base_url, digests = sha256sums_distribution_factory(sha256sums="root") + + assert get_sha256sums(base_url) == {(digest, path) for path, digest in digests.items()} + assert get_sha256sums(base_url, "docs/") is None + assert get_sha256sums(base_url, "docs/api/") is None + + +@pytest.mark.parallel +def test_directory(sha256sums_distribution_factory): + base_url, digests = sha256sums_distribution_factory(sha256sums="directory") + + assert get_sha256sums(base_url) == {(digest, path) for path, digest in digests.items()} + assert get_sha256sums(base_url, "docs/") == { + (digests["docs/intro.md"], "intro.md"), + (digests["docs/api/reference.md"], "api/reference.md"), + } + assert get_sha256sums(base_url, "docs/api/") == { + (digests["docs/api/reference.md"], "reference.md") + } + + +@pytest.mark.parallel +def test_publication_defaults_to_repository( + sha256sums_repository_factory, file_publication_factory +): + repository, _ = sha256sums_repository_factory(sha256sums="directory") + + publication = file_publication_factory(repository=repository.pulp_href, sha256sums="root") + assert publication.sha256sums == "root" + + publication = file_publication_factory(repository=repository.pulp_href) + assert publication.sha256sums == "directory" diff --git a/pulp_file/tests/unit/test_sha256sums.py b/pulp_file/tests/unit/test_sha256sums.py new file mode 100644 index 00000000000..8c5b54f8a71 --- /dev/null +++ b/pulp_file/tests/unit/test_sha256sums.py @@ -0,0 +1,52 @@ +import pytest + +from pulp_file.manifest import Entry, Sha256Sums + +ENTRIES = [ + Entry(relative_path="release.iso", digest="a" * 64, size=1), + Entry(relative_path="docs/intro.md", digest="b" * 64, size=2), + Entry(relative_path="docs/api/reference.md", digest="c" * 64, size=3), +] + + +def read_written(written): + return {path: open(local_path).read() for path, local_path in written.items()} + + +def test_root(tmp_path): + written = Sha256Sums().write(ENTRIES, str(tmp_path)) + + assert read_written(written) == { + "SHA256SUMS": ( + f"{'a' * 64} release.iso\n" + f"{'b' * 64} docs/intro.md\n" + f"{'c' * 64} docs/api/reference.md\n" + ) + } + + +def test_per_directory(tmp_path): + written = Sha256Sums(per_directory=True).write(ENTRIES, str(tmp_path)) + + assert read_written(written) == { + "SHA256SUMS": ( + f"{'a' * 64} release.iso\n" + f"{'b' * 64} docs/intro.md\n" + f"{'c' * 64} docs/api/reference.md\n" + ), + "docs/SHA256SUMS": f"{'b' * 64} intro.md\n{'c' * 64} api/reference.md\n", + "docs/api/SHA256SUMS": f"{'c' * 64} reference.md\n", + } + + +@pytest.mark.parametrize("per_directory", [False, True]) +def test_no_entries(tmp_path, per_directory): + assert Sha256Sums(per_directory=per_directory).write([], str(tmp_path)) == {} + + +def test_entries_without_digest_are_skipped(tmp_path): + entries = ENTRIES + [Entry(relative_path="docs/ondemand.md", digest=None, size=4)] + + written = Sha256Sums(per_directory=True).write(entries, str(tmp_path)) + + assert "ondemand.md" not in read_written(written)["docs/SHA256SUMS"] From a4ed7c792cbc022f664fa9003649e15b400219ed Mon Sep 17 00:00:00 2001 From: Aliaksei Klimau Date: Fri, 2 Oct 2026 13:54:50 +0200 Subject: [PATCH 2/2] Fix SHA256SUMS manifest collision and close files after writing Co-Authored-By: Claude Opus 5 --- pulp_file/app/serializers.py | 31 +++++++++++++ pulp_file/app/viewsets.py | 2 + pulp_file/docs/user/guides/publish-host.md | 4 ++ pulp_file/manifest.py | 22 ++++++---- .../tests/functional/api/test_sha256sums.py | 25 +++++++++++ pulp_file/tests/unit/test_sha256sums.py | 44 ++++++++++++++++--- 6 files changed, 114 insertions(+), 14 deletions(-) diff --git a/pulp_file/app/serializers.py b/pulp_file/app/serializers.py index 0b74aa4ff7d..7d03941ca3d 100644 --- a/pulp_file/app/serializers.py +++ b/pulp_file/app/serializers.py @@ -29,6 +29,23 @@ FileRemote, FileRepository, ) +from pulp_file.manifest import Sha256Sums + + +def validate_manifest_sha256sums(manifest, sha256sums): + """ + Validate that the manifest filename does not collide with a generated SHA256SUMS file. + + Raises: + serializers.ValidationError: If the manifest would be published at a generated path. + + """ + if sha256sums != FileRepository.SHA256SUMS_DISABLED and manifest == Sha256Sums.FILENAME: + raise serializers.ValidationError( + _("The manifest cannot be named {name} while SHA256SUMS files are generated.").format( + name=Sha256Sums.FILENAME + ) + ) class FileContentSerializer(SingleArtifactContentUploadSerializer, ContentChecksumSerializer): @@ -159,6 +176,20 @@ class FileRepositorySerializer(RepositorySerializer): read_only=True, ) + def validate(self, data): + """Validate the FileRepository data.""" + data = super().validate(data) + + validate_manifest_sha256sums( + data.get("manifest", getattr(self.instance, "manifest", None)), + data.get( + "sha256sums", + getattr(self.instance, "sha256sums", FileRepository.SHA256SUMS_DISABLED), + ), + ) + + return data + class Meta: fields = RepositorySerializer.Meta.fields + ( "autopublish", diff --git a/pulp_file/app/viewsets.py b/pulp_file/app/viewsets.py index 41c7180fab5..fe77e0617ba 100644 --- a/pulp_file/app/viewsets.py +++ b/pulp_file/app/viewsets.py @@ -55,6 +55,7 @@ FileRemoteSerializer, FileRepositorySerializer, FileRepositorySyncURLSerializer, + validate_manifest_sha256sums, ) @@ -577,6 +578,7 @@ def create(self, request, **kwargs): sha256sums = serializer.validated_data.get("sha256sums") if sha256sums is None: sha256sums = repository_version.repository.cast().sha256sums + validate_manifest_sha256sums(manifest, sha256sums) task_kwargs = { "repository_version_pk": str(repository_version.pk), diff --git a/pulp_file/docs/user/guides/publish-host.md b/pulp_file/docs/user/guides/publish-host.md index 134a69ffe5b..6f3f38c3bee 100644 --- a/pulp_file/docs/user/guides/publish-host.md +++ b/pulp_file/docs/user/guides/publish-host.md @@ -130,3 +130,7 @@ http POST :/pulp/api/v3/publications/file/file/ repository=$REPO_HREF sha256sums Consumers verify what they downloaded by running `sha256sum -c SHA256SUMS` in the directory they downloaded it into. Content that Pulp has not downloaded yet has no digest to publish and is left out. + +The generated files take the `SHA256SUMS` paths for themselves, so the manifest cannot be named +`SHA256SUMS` while they are generated, and a repository already holding content at one of those +paths cannot be published. diff --git a/pulp_file/manifest.py b/pulp_file/manifest.py index 156e0bbd3b8..6cf2224583e 100644 --- a/pulp_file/manifest.py +++ b/pulp_file/manifest.py @@ -185,27 +185,31 @@ def write(self, entries, dest_dir): dict: The relative path to publish each file at, mapped to its path on disk. """ + written = {} open_files = {} try: - for entry in entries: + # Sorting keeps each directory's entries contiguous, so its file can be closed on the way out. + for entry in sorted(entries, key=lambda entry: entry.relative_path): if not entry.digest: continue - for directory in self._directories(entry.relative_path): - sums_path = os.path.join(directory, self.FILENAME) - if sums_path not in open_files: - open_files[sums_path] = NamedTemporaryFile( - mode="w", dir=dest_dir, delete=False - ) + directories = list(self._directories(entry.relative_path)) + for directory in set(open_files).difference(directories): + open_files.pop(directory).close() + for directory in directories: + if directory not in open_files: + fp = NamedTemporaryFile(mode="w", dir=dest_dir, delete=False) + open_files[directory] = fp + written[os.path.join(directory, self.FILENAME)] = fp.name if directory: name = entry.relative_path[len(directory) + 1 :] else: name = entry.relative_path - open_files[sums_path].write(f"{entry.digest} {name}\n") + open_files[directory].write(f"{entry.digest} {name}\n") finally: for fp in open_files.values(): fp.close() - return {sums_path: fp.name for sums_path, fp in open_files.items()} + return written def _directories(self, relative_path): """ diff --git a/pulp_file/tests/functional/api/test_sha256sums.py b/pulp_file/tests/functional/api/test_sha256sums.py index 816b734fff1..3a9b1f878e1 100644 --- a/pulp_file/tests/functional/api/test_sha256sums.py +++ b/pulp_file/tests/functional/api/test_sha256sums.py @@ -89,3 +89,28 @@ def test_publication_defaults_to_repository( publication = file_publication_factory(repository=repository.pulp_href) assert publication.sha256sums == "directory" + + +@pytest.mark.parallel +def test_repository_manifest_cannot_collide(file_bindings, file_repository_factory): + with pytest.raises(file_bindings.ApiException) as exc: + file_repository_factory(manifest="SHA256SUMS", sha256sums="root") + assert exc.value.status == 400 + + repository = file_repository_factory(sha256sums="root") + with pytest.raises(file_bindings.ApiException) as exc: + file_bindings.RepositoriesFileApi.partial_update( + repository.pulp_href, {"manifest": "SHA256SUMS"} + ) + assert exc.value.status == 400 + + +@pytest.mark.parallel +def test_publication_manifest_cannot_collide(file_bindings, file_repository_factory): + repository = file_repository_factory(sha256sums="root") + + with pytest.raises(file_bindings.ApiException) as exc: + file_bindings.PublicationsFileApi.create( + {"repository": repository.pulp_href, "manifest": "SHA256SUMS"} + ) + assert exc.value.status == 400 diff --git a/pulp_file/tests/unit/test_sha256sums.py b/pulp_file/tests/unit/test_sha256sums.py index 8c5b54f8a71..aa67ba58a82 100644 --- a/pulp_file/tests/unit/test_sha256sums.py +++ b/pulp_file/tests/unit/test_sha256sums.py @@ -1,5 +1,6 @@ import pytest +from pulp_file import manifest as manifest_module from pulp_file.manifest import Entry, Sha256Sums ENTRIES = [ @@ -18,9 +19,9 @@ def test_root(tmp_path): assert read_written(written) == { "SHA256SUMS": ( - f"{'a' * 64} release.iso\n" - f"{'b' * 64} docs/intro.md\n" f"{'c' * 64} docs/api/reference.md\n" + f"{'b' * 64} docs/intro.md\n" + f"{'a' * 64} release.iso\n" ) } @@ -30,11 +31,11 @@ def test_per_directory(tmp_path): assert read_written(written) == { "SHA256SUMS": ( - f"{'a' * 64} release.iso\n" - f"{'b' * 64} docs/intro.md\n" f"{'c' * 64} docs/api/reference.md\n" + f"{'b' * 64} docs/intro.md\n" + f"{'a' * 64} release.iso\n" ), - "docs/SHA256SUMS": f"{'b' * 64} intro.md\n{'c' * 64} api/reference.md\n", + "docs/SHA256SUMS": f"{'c' * 64} api/reference.md\n{'b' * 64} intro.md\n", "docs/api/SHA256SUMS": f"{'c' * 64} reference.md\n", } @@ -50,3 +51,36 @@ def test_entries_without_digest_are_skipped(tmp_path): written = Sha256Sums(per_directory=True).write(entries, str(tmp_path)) assert "ondemand.md" not in read_written(written)["docs/SHA256SUMS"] + + +def test_open_files_are_bound_by_depth(tmp_path, monkeypatch): + entries = [ + Entry(relative_path=f"dir{n}/sub/file.txt", digest=f"{n:064}", size=1) for n in range(100) + ] + open_files = 0 + high_water_mark = 0 + named_temporary_file = manifest_module.NamedTemporaryFile + + def counting_named_temporary_file(*args, **kwargs): + nonlocal open_files, high_water_mark + fp = named_temporary_file(*args, **kwargs) + close = fp.close + open_files += 1 + high_water_mark = max(high_water_mark, open_files) + + def counting_close(): + nonlocal open_files + if not fp.closed: + open_files -= 1 + close() + + fp.close = counting_close + return fp + + monkeypatch.setattr(manifest_module, "NamedTemporaryFile", counting_named_temporary_file) + + written = Sha256Sums(per_directory=True).write(entries, str(tmp_path)) + + assert len(written) == 201 + assert open_files == 0 + assert high_water_mark == 3