From a9b4f6c81c6e6a3db1bb38791d5a46fcfb34bbaf Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 12 Nov 2026 18:12:46 +0100 Subject: [PATCH 1/2] deps: Bump buck2 openSUSE publishes sha512 checksums for its repository metadata and packages, but the pinned buck2 only verifies a download against a sha1 or sha256. The 2026-11-12 release accepts a sha512 in download_file(), so let's bump to it. Signed-off-by: Daan De Meyer --- examples/image/expected.json | 44 ++++++++++++++++++------------------ tine.lock.json | 24 ++++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/examples/image/expected.json b/examples/image/expected.json index 92e5a631..2a29c194 100644 --- a/examples/image/expected.json +++ b/examples/image/expected.json @@ -1,78 +1,78 @@ { "arm64": { "boot-demo-preloaded.fedora": { - "sha256": "eb0291105e6dda37fbe5b77bf603ece5c46ef75077238949aa0faa1191a4dacf" + "sha256": "340da52df8427a60291732dce6556ee8e22f60989ca4bd859207c61f1fe6e494" }, "boot-demo.fedora": { - "sha256": "e3377bc790d7f40a25bd0126b5d18904d29bcc9cbb287047113019845d971b1f" + "sha256": "350aa0af7630242ded49d7e4a351eaa47df14f3978570f18cd82d38e184ffd97" }, "boot-demo.fedora[boot][initrd]": { - "sha256": "0658f07f0ba724f1531c7e7a18b05730399747996d2640e8356da7ba6098452b" + "sha256": "da8021ec0980701daded7760dc958405b507f797a9fc723f9d80949f513bdaab" }, "boot-demo.fedora[boot][kernel]": { "sha256": "f53b29f914a65d1a774bb1faa9fc569263792da86ef8367cb17a27e4c718c81d" }, "boot-demo.fedora[boot][uki]": { - "sha256": "49e38b1fba57b2bb1d9b6d84482b6660f761541c5c7e40d0f62771e6c551c2ad" + "sha256": "855870e52e130974905aa68626426bcfa2c8b6e7692abe3984b635526040df0f" }, "boot-demo.fedora[disk-manifest]": { - "sha256": "be8c8333741efd20eb61b061aeba2105bc7e5fe63755fbe350d3709326bd585e" + "sha256": "6ef84fe095ed130f228e36f3bfa1d917a9019e7cc7912433d1015c6554efcf8e" }, "boot-demo.fedora[manifests][esp]": { - "sha256": "29b830548a2783755d2c469d2ba975a0ab9bc6202b4c289b77a9882b5daf18b2" + "sha256": "e0f42f931767170463082ea0ff19e7924e5fe7b01e9acadbbd9a0b4d8e2fb283" }, "boot-demo.fedora[manifests][usr]": { - "sha256": "314c1e4341d5c5ab22d18519c374c76b5001e2718daed02edbc0e854d5b069b6" + "sha256": "58ccd2d523da512b2e5cbbf6acad108c57f7310f16f12f4c0566796943c828e5" }, "boot-demo.fedora[partitions][usr-verity]": { - "sha256": "8ed5698ae50402404172aa5c5856f51c07057caee43ec25d3127cb0ffc1a7ae4" + "sha256": "02d27f36c17888ad31ee5c80091580f8fdfd4a20cb97795f6621ba7e76595f66" }, "boot-demo.fedora[partitions][usr]": { - "sha256": "ea7b0c7bc1944f912dac3b67ad8ddd184623fdde7d772ad7785afd25258706ba" + "sha256": "22307be590cda99d3a843eccc0a2e1a7fab59ca7efaf3fb12bfce7a7b56d18ae" }, "boot-demo.fedora[roothash]": { - "sha256": "ced37915b93727707e1896c17453c06a805d27eaa31704f2165eb188092ff763" + "sha256": "1e0386e68519c4b56c36e4c7846db5bf7930ebb09fe3c628adfa9cd71c78d102" }, "demo-ext.fedora": { - "sha256": "716d43d09b98a868b361cac63e26bb1fc37187e6cabc95e32fdf5bd2febb9b8d" + "sha256": "754a8373fa7555e81e1909671595c2765c11c2e8d9ae94cdada92d007d961a6c" } }, "x86_64": { "boot-demo-preloaded.fedora": { - "sha256": "5639ec39438ad8de3facbc193b55163785b36956abec603061c95dce4229a8d2" + "sha256": "cf46fd213bd1e2d28d1c5b733e9cd37bfeaaad938d28e84e099b4925ac7c77ec" }, "boot-demo.fedora": { - "sha256": "810f4d8cd0ae75dd5d58c09784615409a4c0c5983f44b9a7e68b3e2b758aaab0" + "sha256": "fb890ef14719bbc9da69ada20c33082fe37f3736ec1ca7450e63c142dbc2d7e0" }, "boot-demo.fedora[boot][initrd]": { - "sha256": "92c4e74ec361a81e7dcb0484dfb97ba060f2c3199f790c0524c2811f21306254" + "sha256": "2060bf8d3484bdf0a56026841ea978047f6d561c697de2b37afc92e5862337e2" }, "boot-demo.fedora[boot][kernel]": { "sha256": "d505498dab710cf2acb7c757d7d50592f147fd418d7464bca95415b190d43c87" }, "boot-demo.fedora[boot][uki]": { - "sha256": "1dbd49e1af726cb9023fd1da842d5441736563c5dea426901a98e2b5e22c14cb" + "sha256": "0649cb08e13d61ab2b5c8a60cd8389ff641b651b7cb8c10a081d72c702e75614" }, "boot-demo.fedora[disk-manifest]": { - "sha256": "8c434b3a0a28f6c6eb6e1609bb766258dbf51c5e0b7e2c79b7161df918561713" + "sha256": "27acb3a200b856c2c151bed68b64c2e9794c037c8db3b9f2189861c56c95d604" }, "boot-demo.fedora[manifests][esp]": { - "sha256": "a856a70ea0b09588e28798cf88250f550ae2475f705ed6d84cea88c076b79946" + "sha256": "6e2edb7d0da9d57b1ccbb8b0d15a6cc714a3284690db6290f9637adcfe1c57dc" }, "boot-demo.fedora[manifests][usr]": { - "sha256": "0451b4a974d9376d975cf92902fc3a287e1a94679947cbddaebc540d838ba861" + "sha256": "f7f98dc99e6b95fdf333061ed462886328f2bf2bcc7d71cece75e9dda1a3e06a" }, "boot-demo.fedora[partitions][usr-verity]": { - "sha256": "8baac782ec9d31c05161bbf86cec6afab8b55d41faad269d932fd67c8680338f" + "sha256": "960535c3221db8b1ab983cf0fcec8a15346c4c210bc13081a10469308398ee94" }, "boot-demo.fedora[partitions][usr]": { - "sha256": "17f8616c33bfeabc14e78f9f45d6ef100f7c257fc674324ed9b502132a5fd7a0" + "sha256": "4ee71aff7592bc82acf3a970364d7ff71ff03aa2421fb6999d0f6efad5c32184" }, "boot-demo.fedora[roothash]": { - "sha256": "f3dbb593040d1eed578b65d23532032df07162777707ffcf0a2a6f70d0e501f5" + "sha256": "a227570e0aeabcd1fedc54489e84f73de00027e20d2b898cab22dad189df6d6c" }, "demo-ext.fedora": { - "sha256": "fe8308bb979aae3004152ed7acc1190b343c46cc822a54c5c65e7ae2e89934ab" + "sha256": "9f5a1562c7fd01d798132a706191f7eb96e5d201681627477643deaad25444d7" } } } diff --git a/tine.lock.json b/tine.lock.json index 4362b1f4..47418d53 100644 --- a/tine.lock.json +++ b/tine.lock.json @@ -1,38 +1,38 @@ { "buck2": { "repository": "daandemeyer/buck2", - "release": "2026-11-05-10.50.24", + "release": "2026-11-12-15.32.16", "platforms": { "Linux-aarch64": { "artifact": "buck2-aarch64-unknown-linux-musl.zst", - "sha256": "bf3af74b22f94f2e0fd9092be987a5360c7093d5e94c68843a77a23475eb996b", - "size": 39031665 + "sha256": "2e687aebd45750b3ab0b3af6d32e4561d66b3481b2bc59ecfdca014c7e086f34", + "size": 40177849 }, "Linux-riscv64": { "artifact": "buck2-riscv64gc-unknown-linux-gnu.zst", - "sha256": "6ddde8e5d85014cc75148f1e058a90255ae78c9c05a8e12dc01466a6f38afd96", - "size": 41100535 + "sha256": "575483be9f62d46316b4c4485cdbb1543b0e5590da6e82b9ac1d903af624da95", + "size": 51887563 }, "Linux-x86_64": { "artifact": "buck2-x86_64-unknown-linux-musl.zst", - "sha256": "75463b17acd9fc2c47876f68ecb1233192398e8d4fedb119cb98962a590e71e2", - "size": 40863186 + "sha256": "795f5581891dc912bad2249c64f5c4d6605cbad652b8356b11d89b36bb5fb690", + "size": 42097357 } } }, "starlark-fmt": { "repository": "daandemeyer/buck2", - "release": "2026-11-05-10.50.24", + "release": "2026-11-12-15.32.16", "platforms": { "arm64": { "artifact": "starlark_fmt-aarch64-unknown-linux-musl.zst", - "sha256": "ea1fffc4250565dc38c5143c4219f5b237c2cc0d2b2ed53c3ee722fe9d06a987", - "size": 2633439 + "sha256": "11ca274966a815beab71c70be31b320fe9eb37d48d60bc9fead6675cc56346f3", + "size": 2671442 }, "x86_64": { "artifact": "starlark_fmt-x86_64-unknown-linux-musl.zst", - "sha256": "b79ba77a6bc62b66b2f61cf65ff3d0c64c0ecad4be776d861f91211625b705ab", - "size": 2759894 + "sha256": "53326dd705ee09219eea86c2a7f20b725bad8af56b37f8ad5e76a16466653b6e", + "size": 2796879 } } }, From 96edae17a400a7d4f275cb9a7773d17dc6517c43 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 12 Nov 2026 18:17:24 +0100 Subject: [PATCH 2/2] package: Accept sha512 digests for repository files openSUSE states a sha512 for every package and every repomd stream, and no sha256. The snapshot, the transaction and the package pool only accept a sha256, so we can't pin an openSUSE repository. Let's key a package by its sha256 or its sha512, and tell a sha256 from a sha512 by its length. A metadata file then contains whichever digest its repository states, and download_file() checks that digest. Signed-off-by: Daan De Meyer Signed-off-by: Martin Pitt --- package/repository.bzl | 15 ++++-- package/snapshotter.py | 38 +++++++++---- package/snapshotter_test.py | 27 +++++++++- package/transaction.py | 6 ++- package/transaction_test.py | 17 ++++++ package_system/rpm/BUCK | 7 +++ package_system/rpm/plan.py | 4 +- package_system/rpm/snapshot.py | 23 +++++--- package_system/rpm/snapshot_test.py | 83 +++++++++++++++++++++++++++++ 9 files changed, 193 insertions(+), 27 deletions(-) create mode 100644 package_system/rpm/snapshot_test.py diff --git a/package/repository.bzl b/package/repository.bzl index 4bae5934..5e77166e 100644 --- a/package/repository.bzl +++ b/package/repository.bzl @@ -119,6 +119,10 @@ def _retained_metadata(id: str, box_locks: list[ArtifactValue], current: list, p generations[json.encode(files)] = entry return [generations[key] for key in sorted(generations)] +# A package's checksum doesn't name its algorithm, so the length of the checksum identifies the algorithm. +# Keep this in sync with DIGEST_LENGTHS in package/snapshotter.py. +_DIGEST_ALGORITHMS = {64: "sha256", 128: "sha512"} + def _download(actions: AnalysisActions, arch: str, path: str, file: dict) -> Artifact: # The path of a content-based output contains no configuration. The configurations of one repository # for two architectures download an identical file, such as a noarch package, to the same path. Two @@ -126,7 +130,7 @@ def _download(actions: AnalysisActions, arch: str, path: str, file: dict) -> Art out = actions.declare_output(arch, path) # A repository may serve a file larger than a Starlark i32, which arrives as a float. - actions.download_file(out, file["url"], sha256 = file["sha256"], size_bytes = int(file["size"])) + actions.download_file(out, file["url"], sha256 = file.get("sha256"), sha512 = file.get("sha512"), size_bytes = int(file["size"])) return out def _materialize_metadata_impl( @@ -187,7 +191,7 @@ def _materialize_package_pool_impl( pool = {} for checksum, package in pool_transports(id, baseurl, snapshot, box_locks).items(): url = package["url"] - artifact = _download(actions, arch, "packages/" + checksum + suffix, {"sha256": checksum, "size": package["size"], "url": url}) + artifact = _download(actions, arch, "packages/" + checksum + suffix, {_DIGEST_ALGORITHMS[len(checksum)]: checksum, "size": package["size"], "url": url}) pool[checksum] = PackageArtifactInfo(artifact = artifact, name = url.rsplit("/", 1)[-1]) return [PackagePoolValueInfo(packages = pool)] @@ -773,7 +777,12 @@ def _select_package_artifacts_impl( package_id = entry["package_id"] if type(rid) != type("") or not rid or type(package_id) != type("") or not package_id: fail("transaction entry has invalid repo/package_id: {}".format(entry)) - if type(checksum) != type("") or len(checksum) != 64 or checksum != checksum.lower() or not _contains_only(checksum, "0123456789abcdef"): + if ( + type(checksum) != type("") + or len(checksum) not in _DIGEST_ALGORITHMS + or checksum != checksum.lower() + or not _contains_only(checksum, "0123456789abcdef") + ): fail("transaction entry has invalid pkg_checksum: {}".format(entry)) # Skip if it's already in the layer below. if below.pop(checksum, None) != None: diff --git a/package/snapshotter.py b/package/snapshotter.py index 2cd4f076..9a192fd0 100644 --- a/package/snapshotter.py +++ b/package/snapshotter.py @@ -17,13 +17,17 @@ import sys from collections.abc import Callable, Mapping from pathlib import Path -from typing import IO, Any, TypedDict +from typing import IO, Any, NotRequired, TypedDict import specs from util import fail, text_destination, urlopen, with_retries AGENT = "tine-snapshot" +# A package's checksum doesn't name its algorithm. The number of hex digits in the checksum identifies +# the algorithm. +DIGEST_LENGTHS = {"sha256": 64, "sha512": 128} + class PackageEntry(TypedDict): location: str @@ -31,11 +35,15 @@ class PackageEntry(TypedDict): class MetadataFile(TypedDict): - """One pinned file, named by where it lands in the materialized repository directory.""" + """One pinned file, named by where it lands in the materialized repository directory. + + It contains the one digest that its repository states for it, `sha256` or `sha512`. + """ out: str url: str - sha256: str + sha256: NotRequired[str] + sha512: NotRequired[str] size: int @@ -110,16 +118,18 @@ def get(self, checksum: str) -> T | None: def pool_checksum(prog: str, package: str) -> str: """The checksum a pool artifact is named by, ahead of its package system's suffix.""" checksum, suffix, _ = Path(package).name.partition(".") - if not suffix or len(checksum) != 64: + if not suffix or len(checksum) not in DIGEST_LENGTHS.values(): fail(f"{prog}: {package} is not named by its checksum") return checksum.lower() -def checksum(rid: str, what: str, value: str | None) -> str: - """Normalize and validate a sha256 a repository states about its own contents.""" +def checksum(rid: str, what: str, value: str | None, algorithm: str = "sha256") -> str: + """Normalize and validate a digest a repository states about its own contents.""" digest = "" if value is None else value.strip().lower() - if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest): - fail(f"{rid}: {what} has invalid sha256 {digest!r}") + if len(digest) != DIGEST_LENGTHS[algorithm] or any( + character not in "0123456789abcdef" for character in digest + ): + fail(f"{rid}: {what} has invalid {algorithm} {digest!r}") return digest @@ -136,11 +146,12 @@ def download( *, size: int | None = None, sha256: str | None = None, + sha512: str | None = None, ) -> tuple[str, int]: - """Stream one file into `output`, returning its digest and size. + """Stream one file into `output`, returning its sha256 and size. - Where the repository has already stated both, they are checked, and an oversized response is - cut off as it arrives rather than after an unbounded endpoint has filled the disk. + Where the repository has already stated a size or digests, they are checked, and an oversized + response is cut off as it arrives rather than after an unbounded endpoint has filled the disk. """ def fetch() -> tuple[str, int]: @@ -148,6 +159,7 @@ def fetch() -> tuple[str, int]: output.seek(0) output.truncate() digest = hashlib.sha256() + stated = hashlib.sha512() if sha512 is not None else None total = 0 with urlopen(url, agent=AGENT, timeout=_TIMEOUT) as response: try: @@ -156,6 +168,8 @@ def fetch() -> tuple[str, int]: if size is not None and total > size: fail(f"{rid}: {what} is larger than the stated {size} bytes") digest.update(chunk) + if stated is not None: + stated.update(chunk) output.write(chunk) except http.client.IncompleteRead as error: raise ConnectionError(f"the transfer ended early: {error}") from error @@ -167,6 +181,8 @@ def fetch() -> tuple[str, int]: fail(f"{rid}: {what} is {total} bytes, not the stated {size}") if sha256 is not None and digest.hexdigest() != sha256: fail(f"{rid}: {what} does not match the checksum stated for it") + if stated is not None and stated.hexdigest() != sha512: + fail(f"{rid}: {what} does not match the checksum stated for it") return digest.hexdigest(), total return with_retries(f"{rid}: {url}", fetch) diff --git a/package/snapshotter_test.py b/package/snapshotter_test.py index 94eae0cc..e5058c00 100644 --- a/package/snapshotter_test.py +++ b/package/snapshotter_test.py @@ -8,6 +8,7 @@ buck test tine//package:test """ +import hashlib import http.client import io import unittest @@ -43,9 +44,11 @@ def setUp(self) -> None: self.enterContext(mock.patch("time.sleep")) self.output = io.BytesIO() - def download(self, *responses: Response) -> None: + def download(self, *responses: Response, sha512: str | None = None) -> None: with mock.patch.object(snapshotter, "urlopen", side_effect=responses): - snapshotter.download("repo", "archive", "https://repo.invalid/archive", self.output) + snapshotter.download( + "repo", "archive", "https://repo.invalid/archive", self.output, sha512=sha512 + ) def test_a_cut_off_chunked_transfer_is_retried(self) -> None: # http.client raises IncompleteRead when a chunked transfer ends early. download() must retry @@ -65,6 +68,26 @@ def test_a_transfer_that_always_ends_early_fails(self) -> None: with self.assertRaisesRegex(ConnectionError, "ended 4 bytes early"): self.download(*(Response(b"arch", missing=4) for _ in range(4))) + def test_a_stated_sha512_is_checked(self) -> None: + # openSUSE states a sha512 for each file of its repository metadata. download() must accept + # content that matches the sha512 and fail on content that does not. + self.download(Response(b"archive\n"), sha512=hashlib.sha512(b"archive\n").hexdigest()) + self.assertEqual(self.output.getvalue(), b"archive\n") + with self.assertRaisesRegex(SystemExit, "does not match the checksum"): + self.download(Response(b"archive\n"), sha512=hashlib.sha512(b"other\n").hexdigest()) + + +class Checksum(unittest.TestCase): + def test_the_algorithm_sets_the_length(self) -> None: + # checksum() must accept a digest of the length of its algorithm, store it in lower case and + # refuse a digest of another algorithm's length. + self.assertEqual(snapshotter.checksum("repo", "file", "AB" * 64, "sha512"), "ab" * 64) + self.assertEqual(snapshotter.checksum("repo", "file", "ab" * 32), "ab" * 32) + with self.assertRaisesRegex(SystemExit, "invalid sha256"): + snapshotter.checksum("repo", "file", "ab" * 64) + with self.assertRaisesRegex(SystemExit, "invalid sha512"): + snapshotter.checksum("repo", "file", "ab" * 32, "sha512") + if __name__ == "__main__": unittest.main() diff --git a/package/transaction.py b/package/transaction.py index ee6632a1..e04bed0b 100644 --- a/package/transaction.py +++ b/package/transaction.py @@ -83,8 +83,10 @@ def load_repositories(spec: Spec, *, absolute: bool = False) -> list[Repository] def checksum(what: str, value: str) -> str: """Check that a package's digest is one a pool can be keyed by.""" digest = value.lower() - if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest): - fail(f"{what} has invalid sha256 {value!r}") + if len(digest) not in snapshotter.DIGEST_LENGTHS.values() or any( + character not in "0123456789abcdef" for character in digest + ): + fail(f"{what} has invalid sha256 or sha512 {value!r}") return digest diff --git a/package/transaction_test.py b/package/transaction_test.py index 29419ba8..09a9bc19 100644 --- a/package/transaction_test.py +++ b/package/transaction_test.py @@ -71,5 +71,22 @@ def test_records_the_metadata_a_vouching_repository_was_resolved_against(self) - ) +class TestEntry(unittest.TestCase): + REPOSITORY = transaction.Repository("repo", Path("repo"), 1, "https://mirror.invalid/repo") + + def test_a_package_is_keyed_by_its_sha256_or_sha512(self) -> None: + # Fedora, Arch and Debian state a sha256 for each package and openSUSE a sha512. entry() + # must accept both digests as the key of a package and store them in lower case. + for digest in ("AB" * 32, "AB" * 64): + package = transaction.entry("pkg", self.REPOSITORY, digest, "pkg.rpm", size=1) + self.assertEqual(package["pkg_checksum"], digest.lower()) + + def test_a_digest_of_another_length_fails(self) -> None: + # Buck can't check a download against a digest of another length, such as a sha1, so + # entry() must refuse it. + with self.assertRaisesRegex(SystemExit, "invalid sha256 or sha512"): + transaction.entry("pkg", self.REPOSITORY, "ab" * 20, "pkg.rpm", size=1) + + if __name__ == "__main__": unittest.main() diff --git a/package_system/rpm/BUCK b/package_system/rpm/BUCK index 2b9c8ab2..93a4b745 100644 --- a/package_system/rpm/BUCK +++ b/package_system/rpm/BUCK @@ -76,6 +76,13 @@ python.binary( main = "snapshot.py", ) +python.test( + name = "snapshot-test", + box = "//tools:dev.box", + srcs = ["snapshot.py", "snapshot_test.py"], + deps = ["//:util", "//package:href", "//package:snapshotter"], +) + python.library(name = "rpmkeys", srcs = ["rpmkeys.py"]) python.binary( diff --git a/package_system/rpm/plan.py b/package_system/rpm/plan.py index 9045461d..856e7a9e 100755 --- a/package_system/rpm/plan.py +++ b/package_system/rpm/plan.py @@ -195,8 +195,8 @@ def plan( if pkg.get_arch() in MULTILIB_ARCHES: fail(f"refusing multilib package {pkg.get_nevra()} (32-bit in a {arch} closure)") chk = pkg.get_checksum() - if chk.get_type_str() != "sha256": - fail(f"expected sha256 repodata checksum for {pkg.get_nevra()}") + if chk.get_type_str() not in ("sha256", "sha512"): + fail(f"expected a sha256 or sha512 repodata checksum for {pkg.get_nevra()}") repo = repositories[pkg.get_repo_id()] resolved.append( transaction.entry( diff --git a/package_system/rpm/snapshot.py b/package_system/rpm/snapshot.py index 2efb935e..6e491330 100644 --- a/package_system/rpm/snapshot.py +++ b/package_system/rpm/snapshot.py @@ -83,9 +83,10 @@ def _parse_primary(rid: str, source: BinaryReader) -> dict[str, PackageEntry]: size = element.find(f"{{{_PRIMARY_NS}}}size") if checksum is None or checksum.text is None: fail(f"{rid}: primary package {count} lacks a checksum") - if checksum.get("type") != "sha256" or checksum.get("pkgid") != "YES": - fail(f"{rid}: primary package {count} does not have a sha256 pkgid") - pkgid = snapshotter.checksum(rid, f"primary package {count} pkgid", checksum.text) + algorithm = checksum.get("type") + if algorithm not in snapshotter.DIGEST_LENGTHS or checksum.get("pkgid") != "YES": + fail(f"{rid}: primary package {count} does not have a sha256 or sha512 pkgid") + pkgid = snapshotter.checksum(rid, f"primary package {count} pkgid", checksum.text, algorithm) href = _location_href(rid, f"primary package {pkgid}", location) @@ -108,7 +109,8 @@ def _load_package_index(rid: str, stream: MetadataFile) -> dict[str, PackageEntr stream["url"], compressed, size=int(stream["size"]), - sha256=stream["sha256"], + sha256=stream.get("sha256"), + sha512=stream.get("sha512"), ) compressed.seek(0) magic = compressed.read(MAGIC) @@ -132,7 +134,10 @@ def _repository_stream( ) -> MetadataFile: """Decode and validate one retained repomd data record.""" location = data.find(f"{{{_REPOMD_NS}}}location") + # Fedora states a sha256 for each stream, and openSUSE states a sha512. checksum = data.find(f"{{{_REPOMD_NS}}}checksum[@type='sha256']") + if checksum is None: + checksum = data.find(f"{{{_REPOMD_NS}}}checksum[@type='sha512']") size = data.find(f"{{{_REPOMD_NS}}}size") if ( location is None @@ -142,16 +147,20 @@ def _repository_stream( or size is None or size.text is None ): - fail(f"{rid}: {stream_type} record lacks a location, sha256 checksum, or size") + fail(f"{rid}: {stream_type} record lacks a location, sha256 or sha512 checksum, or size") href = _location_href(rid, f"{stream_type} stream", location) - return MetadataFile( + stream = MetadataFile( out=f"repodata/{PurePosixPath(href).name}", url=baseurl + href, - sha256=snapshotter.checksum(rid, f"{stream_type} stream", checksum.text), # Recording the compressed size avoids an unpinned HEAD request later. size=_metadata_int(rid, f"{stream_type} stream", size.text, minimum=1), ) + if checksum.get("type") == "sha512": + stream["sha512"] = snapshotter.checksum(rid, f"{stream_type} stream", checksum.text, "sha512") + else: + stream["sha256"] = snapshotter.checksum(rid, f"{stream_type} stream", checksum.text) + return stream def snapshot_repodata(rid: str, baseurl: str) -> RepositorySnapshot: diff --git a/package_system/rpm/snapshot_test.py b/package_system/rpm/snapshot_test.py new file mode 100644 index 00000000..8591d577 --- /dev/null +++ b/package_system/rpm/snapshot_test.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Tests for the parsing of rpm-md repository metadata. + +buck test tine//package_system/rpm:snapshot-test +""" + +import io +import unittest +import xml.etree.ElementTree as ET + +import snapshot + +SHA256 = "ab" * 32 +SHA512 = "cd" * 64 + + +def record(algorithm: str, digest: str) -> ET.Element: + return ET.fromstring( + f'' + f'{digest}' + f'' + f"42" + f"" + ) + + +def primary(algorithm: str, digest: str) -> io.BytesIO: + return io.BytesIO( + f'' + f'' + f'{digest}' + f'' + f'' + f"" + f"".encode() + ) + + +class Stream(unittest.TestCase): + def test_a_stream_keeps_the_digest_its_record_states(self) -> None: + # Fedora states a sha256 for each stream of its repomd.xml and openSUSE a sha512. The pinned + # stream must carry that digest under the name of its algorithm, and no other digest. + for algorithm, digest in (("sha256", SHA256), ("sha512", SHA512)): + stream = snapshot._repository_stream( + "repo", "https://repo.invalid/", "primary", record(algorithm, digest) + ) + self.assertEqual( + stream, + { + "out": f"repodata/{digest}-primary.xml.zst", + "url": f"https://repo.invalid/repodata/{digest}-primary.xml.zst", + "size": 42, + algorithm: digest, + }, + ) + + def test_a_stream_without_a_known_digest_fails(self) -> None: + # Buck can't check a download against a sha1. The snapshot must fail on a record that + # states only a sha1. + with self.assertRaisesRegex(SystemExit, "sha256 or sha512 checksum"): + snapshot._repository_stream( + "repo", "https://repo.invalid/", "primary", record("sha1", "ab" * 20) + ) + + +class Primary(unittest.TestCase): + def test_a_package_is_keyed_by_its_pkgid(self) -> None: + # The snapshot keys a package by its pkgid, which openSUSE states as a sha512 and Fedora as a + # sha256. The sha256 and the sha512 must each become the key of the package's entry. + for algorithm, digest in (("sha256", SHA256), ("sha512", SHA512)): + packages = snapshot._parse_primary("repo", primary(algorithm, digest)) + self.assertEqual(packages, {digest: {"location": "x86_64/pkg-1-1.x86_64.rpm", "size": 7}}) + + def test_a_pkgid_of_the_wrong_length_fails(self) -> None: + # A pkgid whose length does not match its stated algorithm is corrupt metadata. + with self.assertRaisesRegex(SystemExit, "invalid sha512"): + snapshot._parse_primary("repo", primary("sha512", SHA256)) + + +if __name__ == "__main__": + unittest.main()