package: Accept sha512 digests for repository files

examples/image/expected.json

@@ -1,78 +1,78 @@
 {
   "arm64": {
     "boot-demo-preloaded.fedora": {
-      "sha256": "eb0291105e6dda37fbe5b77bf603ece5c46ef75077238949aa0faa1191a4dacf"
+      "sha256": "340da52df8427a60291732dce6556ee8e22f60989ca4bd859207c61f1fe6e494"
     },
     "boot-demo.fedora": {
-      "sha256": "e3377bc790d7f40a25bd0126b5d18904d29bcc9cbb287047113019845d971b1f"
+      "sha256": "350aa0af7630242ded49d7e4a351eaa47df14f3978570f18cd82d38e184ffd97"
     },
     "boot-demo.fedora[boot][initrd]": {
-      "sha256": "0658f07f0ba724f1531c7e7a18b05730399747996d2640e8356da7ba6098452b"
+      "sha256": "da8021ec0980701daded7760dc958405b507f797a9fc723f9d80949f513bdaab"
     },
     "boot-demo.fedora[boot][kernel]": {
       "sha256": "f53b29f914a65d1a774bb1faa9fc569263792da86ef8367cb17a27e4c718c81d"
     },
     "boot-demo.fedora[boot][uki]": {
-      "sha256": "49e38b1fba57b2bb1d9b6d84482b6660f761541c5c7e40d0f62771e6c551c2ad"
+      "sha256": "855870e52e130974905aa68626426bcfa2c8b6e7692abe3984b635526040df0f"
     },
     "boot-demo.fedora[disk-manifest]": {
-      "sha256": "be8c8333741efd20eb61b061aeba2105bc7e5fe63755fbe350d3709326bd585e"
+      "sha256": "6ef84fe095ed130f228e36f3bfa1d917a9019e7cc7912433d1015c6554efcf8e"
     },
     "boot-demo.fedora[manifests][esp]": {
-      "sha256": "29b830548a2783755d2c469d2ba975a0ab9bc6202b4c289b77a9882b5daf18b2"
+      "sha256": "e0f42f931767170463082ea0ff19e7924e5fe7b01e9acadbbd9a0b4d8e2fb283"
     },
     "boot-demo.fedora[manifests][usr]": {
-      "sha256": "314c1e4341d5c5ab22d18519c374c76b5001e2718daed02edbc0e854d5b069b6"
+      "sha256": "58ccd2d523da512b2e5cbbf6acad108c57f7310f16f12f4c0566796943c828e5"
     },
     "boot-demo.fedora[partitions][usr-verity]": {
-      "sha256": "8ed5698ae50402404172aa5c5856f51c07057caee43ec25d3127cb0ffc1a7ae4"
+      "sha256": "02d27f36c17888ad31ee5c80091580f8fdfd4a20cb97795f6621ba7e76595f66"
     },
     "boot-demo.fedora[partitions][usr]": {
-      "sha256": "ea7b0c7bc1944f912dac3b67ad8ddd184623fdde7d772ad7785afd25258706ba"
+      "sha256": "22307be590cda99d3a843eccc0a2e1a7fab59ca7efaf3fb12bfce7a7b56d18ae"
     },
     "boot-demo.fedora[roothash]": {
-      "sha256": "ced37915b93727707e1896c17453c06a805d27eaa31704f2165eb188092ff763"
+      "sha256": "1e0386e68519c4b56c36e4c7846db5bf7930ebb09fe3c628adfa9cd71c78d102"
     },
     "demo-ext.fedora": {
-      "sha256": "716d43d09b98a868b361cac63e26bb1fc37187e6cabc95e32fdf5bd2febb9b8d"
+      "sha256": "754a8373fa7555e81e1909671595c2765c11c2e8d9ae94cdada92d007d961a6c"
     }
   },
   "x86_64": {
     "boot-demo-preloaded.fedora": {
-      "sha256": "5639ec39438ad8de3facbc193b55163785b36956abec603061c95dce4229a8d2"
+      "sha256": "cf46fd213bd1e2d28d1c5b733e9cd37bfeaaad938d28e84e099b4925ac7c77ec"
     },
     "boot-demo.fedora": {
-      "sha256": "810f4d8cd0ae75dd5d58c09784615409a4c0c5983f44b9a7e68b3e2b758aaab0"
+      "sha256": "fb890ef14719bbc9da69ada20c33082fe37f3736ec1ca7450e63c142dbc2d7e0"
     },
     "boot-demo.fedora[boot][initrd]": {
-      "sha256": "92c4e74ec361a81e7dcb0484dfb97ba060f2c3199f790c0524c2811f21306254"
+      "sha256": "2060bf8d3484bdf0a56026841ea978047f6d561c697de2b37afc92e5862337e2"
     },
     "boot-demo.fedora[boot][kernel]": {
       "sha256": "d505498dab710cf2acb7c757d7d50592f147fd418d7464bca95415b190d43c87"
     },
     "boot-demo.fedora[boot][uki]": {
-      "sha256": "1dbd49e1af726cb9023fd1da842d5441736563c5dea426901a98e2b5e22c14cb"
+      "sha256": "0649cb08e13d61ab2b5c8a60cd8389ff641b651b7cb8c10a081d72c702e75614"
     },
     "boot-demo.fedora[disk-manifest]": {
-      "sha256": "8c434b3a0a28f6c6eb6e1609bb766258dbf51c5e0b7e2c79b7161df918561713"
+      "sha256": "27acb3a200b856c2c151bed68b64c2e9794c037c8db3b9f2189861c56c95d604"
     },
     "boot-demo.fedora[manifests][esp]": {
-      "sha256": "a856a70ea0b09588e28798cf88250f550ae2475f705ed6d84cea88c076b79946"
+      "sha256": "6e2edb7d0da9d57b1ccbb8b0d15a6cc714a3284690db6290f9637adcfe1c57dc"
     },
     "boot-demo.fedora[manifests][usr]": {
-      "sha256": "0451b4a974d9376d975cf92902fc3a287e1a94679947cbddaebc540d838ba861"
+      "sha256": "f7f98dc99e6b95fdf333061ed462886328f2bf2bcc7d71cece75e9dda1a3e06a"
     },
     "boot-demo.fedora[partitions][usr-verity]": {
-      "sha256": "8baac782ec9d31c05161bbf86cec6afab8b55d41faad269d932fd67c8680338f"
+      "sha256": "960535c3221db8b1ab983cf0fcec8a15346c4c210bc13081a10469308398ee94"
     },
     "boot-demo.fedora[partitions][usr]": {
-      "sha256": "17f8616c33bfeabc14e78f9f45d6ef100f7c257fc674324ed9b502132a5fd7a0"
+      "sha256": "4ee71aff7592bc82acf3a970364d7ff71ff03aa2421fb6999d0f6efad5c32184"
     },
     "boot-demo.fedora[roothash]": {
-      "sha256": "f3dbb593040d1eed578b65d23532032df07162777707ffcf0a2a6f70d0e501f5"
+      "sha256": "a227570e0aeabcd1fedc54489e84f73de00027e20d2b898cab22dad189df6d6c"
     },
     "demo-ext.fedora": {
-      "sha256": "fe8308bb979aae3004152ed7acc1190b343c46cc822a54c5c65e7ae2e89934ab"
+      "sha256": "9f5a1562c7fd01d798132a706191f7eb96e5d201681627477643deaad25444d7"
     }
   }
 }

package/repository.bzl

@@ -119,14 +119,18 @@ def _retained_metadata(id: str, box_locks: list[ArtifactValue], current: list, p
             generations[json.encode(files)] = entry
     return [generations[key] for key in sorted(generations)]
 
+# A package's checksum doesn't name its algorithm, so the length of the checksum identifies the algorithm.
+# Keep this in sync with DIGEST_LENGTHS in package/snapshotter.py.
+_DIGEST_ALGORITHMS = {64: "sha256", 128: "sha512"}
+
 def _download(actions: AnalysisActions, arch: str, path: str, file: dict) -> Artifact:
     # The path of a content-based output contains no configuration. The configurations of one repository
     # for two architectures download an identical file, such as a noarch package, to the same path. Two
     # parallel downloads into one path corrupt the file, so the architecture is part of the path.
     out = actions.declare_output(arch, path)
 
     # A repository may serve a file larger than a Starlark i32, which arrives as a float.
-    actions.download_file(out, file["url"], sha256 = file["sha256"], size_bytes = int(file["size"]))
+    actions.download_file(out, file["url"], sha256 = file.get("sha256"), sha512 = file.get("sha512"), size_bytes = int(file["size"]))
     return out
 
 def _materialize_metadata_impl(
@@ -187,7 +191,7 @@ def _materialize_package_pool_impl(
     pool = {}
     for checksum, package in pool_transports(id, baseurl, snapshot, box_locks).items():
         url = package["url"]
-        artifact = _download(actions, arch, "packages/" + checksum + suffix, {"sha256": checksum, "size": package["size"], "url": url})
+        artifact = _download(actions, arch, "packages/" + checksum + suffix, {_DIGEST_ALGORITHMS[len(checksum)]: checksum, "size": package["size"], "url": url})
         pool[checksum] = PackageArtifactInfo(artifact = artifact, name = url.rsplit("/", 1)[-1])
     return [PackagePoolValueInfo(packages = pool)]
 
@@ -773,7 +777,12 @@ def _select_package_artifacts_impl(
         package_id = entry["package_id"]
         if type(rid) != type("") or not rid or type(package_id) != type("") or not package_id:
             fail("transaction entry has invalid repo/package_id: {}".format(entry))
-        if type(checksum) != type("") or len(checksum) != 64 or checksum != checksum.lower() or not _contains_only(checksum, "0123456789abcdef"):
+        if (
+            type(checksum) != type("")
+            or len(checksum) not in _DIGEST_ALGORITHMS
+            or checksum != checksum.lower()
+            or not _contains_only(checksum, "0123456789abcdef")
+        ):
             fail("transaction entry has invalid pkg_checksum: {}".format(entry))
         # Skip if it's already in the layer below.
         if below.pop(checksum, None) != None:

package/snapshotter.py

@@ -17,25 +17,33 @@
 import sys
 from collections.abc import Callable, Mapping
 from pathlib import Path
-from typing import IO, Any, TypedDict
+from typing import IO, Any, NotRequired, TypedDict
 
 import specs
 from util import fail, text_destination, urlopen, with_retries
 
 AGENT = "tine-snapshot"
 
+# A package's checksum doesn't name its algorithm. The number of hex digits in the checksum identifies
+# the algorithm.
+DIGEST_LENGTHS = {"sha256": 64, "sha512": 128}
+
 
 class PackageEntry(TypedDict):
     location: str
     size: int
 
 
 class MetadataFile(TypedDict):
-    """One pinned file, named by where it lands in the materialized repository directory."""
+    """One pinned file, named by where it lands in the materialized repository directory.
+
+    It contains the one digest that its repository states for it, `sha256` or `sha512`.
+    """
 
     out: str
     url: str
-    sha256: str
+    sha256: NotRequired[str]
+    sha512: NotRequired[str]
     size: int
 
 
@@ -110,16 +118,18 @@ def get(self, checksum: str) -> T | None:
 def pool_checksum(prog: str, package: str) -> str:
     """The checksum a pool artifact is named by, ahead of its package system's suffix."""
     checksum, suffix, _ = Path(package).name.partition(".")
-    if not suffix or len(checksum) != 64:
+    if not suffix or len(checksum) not in DIGEST_LENGTHS.values():
         fail(f"{prog}: {package} is not named by its checksum")
     return checksum.lower()
 
 
-def checksum(rid: str, what: str, value: str | None) -> str:
-    """Normalize and validate a sha256 a repository states about its own contents."""
+def checksum(rid: str, what: str, value: str | None, algorithm: str = "sha256") -> str:
+    """Normalize and validate a digest a repository states about its own contents."""
     digest = "" if value is None else value.strip().lower()
-    if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest):
-        fail(f"{rid}: {what} has invalid sha256 {digest!r}")
+    if len(digest) != DIGEST_LENGTHS[algorithm] or any(
+        character not in "0123456789abcdef" for character in digest
+    ):
+        fail(f"{rid}: {what} has invalid {algorithm} {digest!r}")
     return digest
 
 
@@ -136,18 +146,20 @@ def download(
     *,
     size: int | None = None,
     sha256: str | None = None,
+    sha512: str | None = None,
 ) -> tuple[str, int]:
-    """Stream one file into `output`, returning its digest and size.
+    """Stream one file into `output`, returning its sha256 and size.
 
-    Where the repository has already stated both, they are checked, and an oversized response is
-    cut off as it arrives rather than after an unbounded endpoint has filled the disk.
+    Where the repository has already stated a size or digests, they are checked, and an oversized
+    response is cut off as it arrives rather than after an unbounded endpoint has filled the disk.
     """
 
     def fetch() -> tuple[str, int]:
         # A retried attempt restarts the stream from scratch.
         output.seek(0)
         output.truncate()
         digest = hashlib.sha256()
+        stated = hashlib.sha512() if sha512 is not None else None
         total = 0
         with urlopen(url, agent=AGENT, timeout=_TIMEOUT) as response:
             try:
@@ -156,6 +168,8 @@ def fetch() -> tuple[str, int]:
                     if size is not None and total > size:
                         fail(f"{rid}: {what} is larger than the stated {size} bytes")
                     digest.update(chunk)
+                    if stated is not None:
+                        stated.update(chunk)
                     output.write(chunk)
             except http.client.IncompleteRead as error:
                 raise ConnectionError(f"the transfer ended early: {error}") from error
@@ -167,6 +181,8 @@ def fetch() -> tuple[str, int]:
             fail(f"{rid}: {what} is {total} bytes, not the stated {size}")
         if sha256 is not None and digest.hexdigest() != sha256:
             fail(f"{rid}: {what} does not match the checksum stated for it")
+        if stated is not None and stated.hexdigest() != sha512:
+            fail(f"{rid}: {what} does not match the checksum stated for it")
         return digest.hexdigest(), total
 
     return with_retries(f"{rid}: {url}", fetch)

package/snapshotter_test.py

@@ -8,6 +8,7 @@
 buck test tine//package:test
 """
 
+import hashlib
 import http.client
 import io
 import unittest
@@ -43,9 +44,11 @@ def setUp(self) -> None:
         self.enterContext(mock.patch("time.sleep"))
         self.output = io.BytesIO()
 
-    def download(self, *responses: Response) -> None:
+    def download(self, *responses: Response, sha512: str | None = None) -> None:
         with mock.patch.object(snapshotter, "urlopen", side_effect=responses):
-            snapshotter.download("repo", "archive", "https://repo.invalid/archive", self.output)
+            snapshotter.download(
+                "repo", "archive", "https://repo.invalid/archive", self.output, sha512=sha512
+            )
 
     def test_a_cut_off_chunked_transfer_is_retried(self) -> None:
         # http.client raises IncompleteRead when a chunked transfer ends early. download() must retry
@@ -65,6 +68,26 @@ def test_a_transfer_that_always_ends_early_fails(self) -> None:
         with self.assertRaisesRegex(ConnectionError, "ended 4 bytes early"):
             self.download(*(Response(b"arch", missing=4) for _ in range(4)))
 
+    def test_a_stated_sha512_is_checked(self) -> None:
+        # openSUSE states a sha512 for each file of its repository metadata. download() must accept
+        # content that matches the sha512 and fail on content that does not.
+        self.download(Response(b"archive\n"), sha512=hashlib.sha512(b"archive\n").hexdigest())
+        self.assertEqual(self.output.getvalue(), b"archive\n")
+        with self.assertRaisesRegex(SystemExit, "does not match the checksum"):
+            self.download(Response(b"archive\n"), sha512=hashlib.sha512(b"other\n").hexdigest())
+
+
+class Checksum(unittest.TestCase):
+    def test_the_algorithm_sets_the_length(self) -> None:
+        # checksum() must accept a digest of the length of its algorithm, store it in lower case and
+        # refuse a digest of another algorithm's length.
+        self.assertEqual(snapshotter.checksum("repo", "file", "AB" * 64, "sha512"), "ab" * 64)
+        self.assertEqual(snapshotter.checksum("repo", "file", "ab" * 32), "ab" * 32)
+        with self.assertRaisesRegex(SystemExit, "invalid sha256"):
+            snapshotter.checksum("repo", "file", "ab" * 64)
+        with self.assertRaisesRegex(SystemExit, "invalid sha512"):
+            snapshotter.checksum("repo", "file", "ab" * 32, "sha512")
+
 
 if __name__ == "__main__":
     unittest.main()

package/transaction.py

@@ -83,8 +83,10 @@ def load_repositories(spec: Spec, *, absolute: bool = False) -> list[Repository]
 def checksum(what: str, value: str) -> str:
     """Check that a package's digest is one a pool can be keyed by."""
     digest = value.lower()
-    if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest):
-        fail(f"{what} has invalid sha256 {value!r}")
+    if len(digest) not in snapshotter.DIGEST_LENGTHS.values() or any(
+        character not in "0123456789abcdef" for character in digest
+    ):
+        fail(f"{what} has invalid sha256 or sha512 {value!r}")
     return digest
 
 

package/transaction_test.py

@@ -71,5 +71,22 @@ def test_records_the_metadata_a_vouching_repository_was_resolved_against(self) -
         )
 
 
+class TestEntry(unittest.TestCase):
+    REPOSITORY = transaction.Repository("repo", Path("repo"), 1, "https://mirror.invalid/repo")
+
+    def test_a_package_is_keyed_by_its_sha256_or_sha512(self) -> None:
+        # Fedora, Arch and Debian state a sha256 for each package and openSUSE a sha512. entry()
+        # must accept both digests as the key of a package and store them in lower case.
+        for digest in ("AB" * 32, "AB" * 64):
+            package = transaction.entry("pkg", self.REPOSITORY, digest, "pkg.rpm", size=1)
+            self.assertEqual(package["pkg_checksum"], digest.lower())
+
+    def test_a_digest_of_another_length_fails(self) -> None:
+        # Buck can't check a download against a digest of another length, such as a sha1, so
+        # entry() must refuse it.
+        with self.assertRaisesRegex(SystemExit, "invalid sha256 or sha512"):
+            transaction.entry("pkg", self.REPOSITORY, "ab" * 20, "pkg.rpm", size=1)
+
+
 if __name__ == "__main__":
     unittest.main()

package_system/rpm/BUCK

@@ -76,6 +76,13 @@ python.binary(
     main = "snapshot.py",
 )
 
+python.test(
+    name = "snapshot-test",
+    box = "//tools:dev.box",
+    srcs = ["snapshot.py", "snapshot_test.py"],
+    deps = ["//:util", "//package:href", "//package:snapshotter"],
+)
+
 python.library(name = "rpmkeys", srcs = ["rpmkeys.py"])
 
 python.binary(

package_system/rpm/plan.py

@@ -195,8 +195,8 @@ def plan(
         if pkg.get_arch() in MULTILIB_ARCHES:
             fail(f"refusing multilib package {pkg.get_nevra()} (32-bit in a {arch} closure)")
         chk = pkg.get_checksum()
-        if chk.get_type_str() != "sha256":
-            fail(f"expected sha256 repodata checksum for {pkg.get_nevra()}")
+        if chk.get_type_str() not in ("sha256", "sha512"):
+            fail(f"expected a sha256 or sha512 repodata checksum for {pkg.get_nevra()}")
         repo = repositories[pkg.get_repo_id()]
         resolved.append(
             transaction.entry(