From 72e45659222a4c87a3f890e58e6d3d362f81f711 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 06:18:34 +0000 Subject: [PATCH 1/4] feat(origin): hosted .kpar packages by download_url and checksum qualifiers A pkg:generic/[@]?download_url=[&checksum=:] purl names a .kpar at any http(s) URL (ECMA-427 Annex B.3.5). The archive is fetched as a plain .kpar IRI is, the declared checksums are verified before it is opened, and it installs as lib/-_.kpar with the archive digest in the lock. vcs_url and download_url exclude each other; add writes the canonical form, and .project.json no longer escapes & as \u0026. Co-Authored-By: jason.han --- README.md | 13 +- .../unreleased/download-url-archives.added.md | 1 + cmd/syspi/main.go | 3 + docs/packages.md | 37 +++++- internal/digest/checksum.go | 124 ++++++++++++++++++ internal/digest/checksum_test.go | 55 ++++++++ internal/install/install.go | 44 ++++++- internal/install/install_test.go | 93 +++++++++++++ internal/origin/origin.go | 114 ++++++++++++---- internal/origin/origin_test.go | 72 +++++++++- internal/project/meta.go | 8 +- internal/project/project.go | 8 +- tests/e2e/e2e_test.go | 61 +++++++++ 13 files changed, 598 insertions(+), 35 deletions(-) create mode 100644 changes/unreleased/download-url-archives.added.md create mode 100644 internal/digest/checksum.go create mode 100644 internal/digest/checksum_test.go diff --git a/README.md b/README.md index da22cf2..207c569 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ what package managers install. ## The contract, on one screen - **Identity** — an ECMA-427 Package-URL of type `generic` with the Annex B.3.5 `vcs_url` - qualifier: `pkg:generic/?vcs_url=+:///[@][#]`, + qualifier for a repository (or `download_url` for a hosted `.kpar`, under "Other sources"): + `pkg:generic/?vcs_url=+:///[@][#]`, the qualifier percent-encoded (`+` → `%2B`, `/` → `%2F`, `@` → `%40`, `#` → `%23`, `:` kept), e.g. `pkg:generic/OpenSysML-Extensions-Library?vcs_url=git%2Bhttps:%2F%2Fgithub.com%2FOpen-MBEE%2FOpenSysML-Extensions-Library%401.0.0`. `` is `git`, `svn` or `hg`, `` one of Annex B.3.5's (`git`, `https`, `http`, @@ -44,7 +45,15 @@ what package managers install. `.kpar` SHA-256, and path-traversal, symlink and size bounds on every archive and export. - **Other sources** — a local directory (editable, read in place), a local `.kpar`, a remote `.kpar` URL, OMG standard-library IRIs (`https://www.omg.org/spec/{KerML,SysML}/…/*.kpar`, - recorded but never installed: every reader bundles them). + recorded but never installed: every reader bundles them). A `.kpar` hosted at any http(s) URL + — a release asset, a repository manager's export — is also a Package-URL with the Annex B.3.5 + `download_url` qualifier and an optional `checksum`: + `pkg:generic/[@]?download_url=[&checksum=:[,…]]`, both + percent-encoded (`pkg:generic/tools?checksum=sha256:9f86…&download_url=https:%2F%2Fdist.example.org%2Ftools.kpar`). + The archive is fetched as a plain `.kpar` URL is (netrc, redirects, size bound), the declared + checksums are verified before it is opened (a mismatch installs nothing), `` is the + archive's own and is held to `@` / `versionConstraint`, and the lock's `sha256` is the + archive digest. Details — the grammar, the svn `tags/` convention, the tree-hash algorithm and the lock format — are in [docs/packages.md](docs/packages.md). diff --git a/changes/unreleased/download-url-archives.added.md b/changes/unreleased/download-url-archives.added.md new file mode 100644 index 0000000..c958523 --- /dev/null +++ b/changes/unreleased/download-url-archives.added.md @@ -0,0 +1 @@ +- A `.kpar` hosted at any http(s) URL can be named by an ECMA-427 Package-URL with the Annex B.3.5 `download_url` qualifier and an optional `checksum` (`pkg:generic/[@]?download_url=[&checksum=:[,…]]`, percent-encoded; md5, sha1, sha224, sha256, sha384, sha512 and the sha3 family). The archive is fetched as a plain `.kpar` URL is, every declared checksum is verified before it is opened — a mismatch installs nothing — and it installs as `lib/-_.kpar` with the archive's SHA-256 in the lock; `vcs_url` and `download_url` exclude each other, and `syspi add` writes the canonical form into `.project.json`. diff --git a/cmd/syspi/main.go b/cmd/syspi/main.go index 2ee2eb2..6357b90 100644 --- a/cmd/syspi/main.go +++ b/cmd/syspi/main.go @@ -45,6 +45,9 @@ ECMA-427 Package-URL (vcs: git, svn, hg; transports per Annex B.3.5, plus file): Accepted as input and normalised to it: pkg:git//[@][#] +:///[@][#] +A .kpar hosted at an http(s) URL is its IRI, or a Package-URL with the Annex B.3.5 download_url +qualifier and an optional checksum (:, verified before the archive is opened): + pkg:generic/[@]?download_url=https:%2F%2F%2F[&checksum=sha256:] Run "syspi -h" for the flags of a command. ` diff --git a/docs/packages.md b/docs/packages.md index 09daf1a..52e21dc 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -50,6 +50,41 @@ host (Annex B.3.5), a transport outside the lists above, `bzr` or any other VCS, file — a `.kpar` — as KerML 1.1 §10.3 Table 12 describes `resource`; it is never cloned, and `user:password@` in it is refused (credentials belong in the netrc, never in `.project.json`). +### Hosted archives + +A `.kpar` hosted at an http(s) URL is named either by that URL (a plain IRI, dereferenced as a +project interchange file) or by a Package-URL of type `generic` carrying the Annex B.3.5 +`download_url` qualifier, with an optional `checksum`: + +``` +resource = "pkg:generic/" name [ "@" version ] "?" qualifiers +qualifiers = [ "checksum=" encode(checksums) "&" ] "download_url=" encode(url) (canonical order; any order accepted) +url = "http" [ "s" ] "://" host "/" path [ "?" query ] (no user:password, no "#") +checksums = checksum *( "," checksum ) ("," encoded as %2C) +checksum = algorithm ":" hex (lower case; Annex B.3.5) +algorithm = "md5" | "sha1" | "sha224" | "sha256" | "sha384" | "sha512" + | "sha3-224" | "sha3-256" | "sha3-384" | "sha3-512" +name = the archive's .project.json "name" +version = semantic version the archive's .project.json must carry; an exact constraint +``` + +`vcs_url` and `download_url` exclude each other on one purl; `checksum` is only accepted with +`download_url` (a repository is held by its tree hash). Any URL works — a release asset, a +repository manager's export — and it need not end in `.kpar`; what it serves must be one. The +archive is downloaded as a plain `.kpar` URL is (netrc, redirects, the size bound), every +declared checksum is verified against the downloaded bytes before the archive is opened — a +mismatch is an error and nothing is installed — and an algorithm this tool cannot compute is +refused rather than skipped. The version is the archive's own `.project.json` `version`, checked +against `@` and `versionConstraint` as a plain IRI's is, and the purl `` must be +the archive's `name`. The archive installs as `lib/-_.kpar`; the lock +records the version-free identity `pkg:generic/?download_url=encode(url)` as `resource`, +the URL as `source` and the archive's SHA-256 as `sha256`, whatever algorithms the purl declares. +`syspi add` writes the canonical form (version after the name, `checksum` before `download_url`, +items sorted by algorithm) into `.project.json`; a later run with a declared checksum that +disagrees with the lock or the installed archive fails. Example: + +- `pkg:generic/tools@3.0.0?checksum=sha256:9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08&download_url=https:%2F%2Fdist.example.org%2Freleases%2Facme-tools_3.0.0.kpar` + ## What a package is The tagged tree — the repository root, or `#` within it — is the project directory. It @@ -138,7 +173,7 @@ and the lock as they were. A lock entry wins over the constraint in `.project.js .project.lock.json lib/ mbse4u-sysmod_5.1.5/ exported tree of a tagged repository - acme-tools_3.0.0.kpar archive source, verified and copied as is + acme-tools_3.0.0.kpar archive source (a .kpar path, URL or download_url purl), verified and copied as is ``` The directory name is `-_` (`_` without a publisher), diff --git a/internal/digest/checksum.go b/internal/digest/checksum.go new file mode 100644 index 0000000..c199f29 --- /dev/null +++ b/internal/digest/checksum.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Apache-2.0 + +package digest + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha3" + "crypto/sha512" + "encoding/hex" + "fmt" + "hash" + "io" + "os" + "sort" + "strings" +) + +// Checksum is one ECMA-427 Annex B.3.5 checksum item: a lower-case algorithm +// name and the lower-case hex of its value. +type Checksum struct { + Algorithm, Value string +} + +// String is the item as a checksum qualifier spells it, :. +func (c Checksum) String() string { return c.Algorithm + ":" + c.Value } + +// algorithms are the Annex B.3.5 algorithms this tool can compute. +var algorithms = map[string]func() hash.Hash{ + "md5": md5.New, + "sha1": sha1.New, + "sha224": sha256.New224, + "sha256": sha256.New, + "sha384": sha512.New384, + "sha512": sha512.New, + "sha3-224": func() hash.Hash { return sha3.New224() }, + "sha3-256": func() hash.Hash { return sha3.New256() }, + "sha3-384": func() hash.Hash { return sha3.New384() }, + "sha3-512": func() hash.Hash { return sha3.New512() }, +} + +// Algorithms lists the supported checksum algorithms, sorted. +func Algorithms() []string { + out := make([]string, 0, len(algorithms)) + for a := range algorithms { + out = append(out, a) + } + sort.Strings(out) + return out +} + +// ParseChecksums reads a checksum qualifier value, : items +// separated by commas (Annex B.3.5), normalised to lower case and sorted by +// algorithm. An algorithm this tool cannot compute is an error: a declared +// checksum that cannot be verified is not one that passes. +func ParseChecksums(value string) ([]Checksum, error) { + var out []Checksum + seen := map[string]string{} + for _, item := range strings.Split(value, ",") { + algo, hexValue, ok := strings.Cut(item, ":") + algo, hexValue = strings.ToLower(strings.TrimSpace(algo)), strings.ToLower(strings.TrimSpace(hexValue)) + if !ok || algo == "" || hexValue == "" { + return nil, fmt.Errorf("checksum %q is not :", item) + } + newHash, known := algorithms[algo] + if !known { + return nil, fmt.Errorf("checksum algorithm %q is not supported (%s are)", algo, strings.Join(Algorithms(), ", ")) + } + raw, err := hex.DecodeString(hexValue) + if err != nil { + return nil, fmt.Errorf("checksum %q: the %s value is not hex", item, algo) + } + if want := newHash().Size(); len(raw) != want { + return nil, fmt.Errorf("checksum %q: a %s value is %d hex digits, not %d", item, algo, 2*want, len(hexValue)) + } + if prev, dup := seen[algo]; dup && prev != hexValue { + return nil, fmt.Errorf("checksum declares two different %s values", algo) + } else if dup { + continue + } + seen[algo] = hexValue + out = append(out, Checksum{algo, hexValue}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Algorithm < out[j].Algorithm }) + return out, nil +} + +// FormatChecksums is the qualifier value for the items, comma-separated. +func FormatChecksums(sums []Checksum) string { + parts := make([]string, len(sums)) + for i, c := range sums { + parts[i] = c.String() + } + return strings.Join(parts, ",") +} + +// Verify checks the file's bytes against every declared checksum in one pass +// and reports the first that does not match. +func Verify(path string, sums []Checksum) error { + if len(sums) == 0 { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + hashes := make([]hash.Hash, len(sums)) + writers := make([]io.Writer, len(sums)) + for i, c := range sums { + hashes[i] = algorithms[c.Algorithm]() + writers[i] = hashes[i] + } + if _, err := io.Copy(io.MultiWriter(writers...), f); err != nil { + return err + } + for i, c := range sums { + if got := hex.EncodeToString(hashes[i].Sum(nil)); got != c.Value { + return fmt.Errorf("checksum mismatch: the resource declares %s %s, the archive has %s", c.Algorithm, c.Value, got) + } + } + return nil +} diff --git a/internal/digest/checksum_test.go b/internal/digest/checksum_test.go new file mode 100644 index 0000000..748d9b4 --- /dev/null +++ b/internal/digest/checksum_test.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +package digest + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseChecksums(t *testing.T) { + sha := strings.Repeat("ab", 32) + md := strings.Repeat("CD", 16) + got, err := ParseChecksums("sha256:" + sha + ",md5:" + md + ",sha256:" + strings.ToUpper(sha)) + if err != nil { + t.Fatal(err) + } + if FormatChecksums(got) != "md5:"+strings.ToLower(md)+",sha256:"+sha { + t.Fatalf("got %s", FormatChecksums(got)) + } + for _, tc := range []struct{ in, want string }{ + {"sha256", "is not :"}, + {"sha256:", "is not :"}, + {":" + sha, "is not :"}, + {"crc32:deadbeef", `algorithm "crc32" is not supported`}, + {"sha256:xyz", "is not hex"}, + {"sha256:abcd", "a sha256 value is 64 hex digits, not 4"}, + {"sha256:" + sha + ",sha256:" + strings.Repeat("00", 32), "two different sha256 values"}, + } { + if _, err := ParseChecksums(tc.in); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%q: got %v, want %q", tc.in, err, tc.want) + } + } +} + +func TestVerify(t *testing.T) { + f := filepath.Join(t.TempDir(), "x.kpar") + os.WriteFile(f, []byte("hello"), 0o644) + want, _ := File(f) + sums, err := ParseChecksums("sha256:" + want + ",sha1:aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d,sha3-256:3338be694f50c5f338814986cdf0686453a888b84f424d792af4b9202398f392") + if err != nil { + t.Fatal(err) + } + if err := Verify(f, sums); err != nil { + t.Fatal(err) + } + if err := Verify(f, nil); err != nil { + t.Fatal(err) + } + bad, _ := ParseChecksums("sha256:" + strings.Repeat("00", 32)) + if err := Verify(f, bad); err == nil || !strings.Contains(err.Error(), "checksum mismatch: the resource declares sha256 "+strings.Repeat("00", 32)+", the archive has "+want) { + t.Fatalf("got %v", err) + } +} diff --git a/internal/install/install.go b/internal/install/install.go index 90d05ff..44cfc33 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -440,8 +440,7 @@ func (r *resolver) reuseLocked(old *lock.Entry) (*lock.Entry, bool, error) { if old == nil { return nil, false, nil } - stem := (&project.Project{Name: old.Name, Publisher: old.Publisher, Version: old.Version}).InstallName() - dst := filepath.Join(r.libDir, stem+project.ArchiveExt) + dst := r.lockedArchivePath(old) cur, err := digest.File(dst) if r.opts.LockOnly || (err == nil && cur == old.SHA256) { if !r.opts.LockOnly { @@ -457,6 +456,31 @@ func (r *resolver) reuseLocked(old *lock.Entry) (*lock.Entry, bool, error) { return nil, false, nil } +// verifyLocked holds a reused lock entry to the checksums a hosted .kpar's +// resource declares: a sha256 item against the lock's digest, every item +// against the installed archive when it is present. +func (r *resolver) verifyLocked(o *origin.Origin, old *lock.Entry) error { + if len(o.Checksums) == 0 { + return nil + } + for _, c := range o.Checksums { + if c.Algorithm == "sha256" && c.Value != old.SHA256 { + return fmt.Errorf("checksum mismatch: the resource declares sha256 %s, the lock records %s; remove the lock entry to resolve it again", c.Value, old.SHA256) + } + } + dst := r.lockedArchivePath(old) + if _, err := os.Stat(dst); err != nil { + return nil + } + return digest.Verify(dst, o.Checksums) +} + +// lockedArchivePath is where a locked archive entry is installed under lib/. +func (r *resolver) lockedArchivePath(old *lock.Entry) string { + stem := (&project.Project{Name: old.Name, Publisher: old.Publisher, Version: old.Version}).InstallName() + return filepath.Join(r.libDir, stem+project.ArchiveExt) +} + // requeueLocked queues the usages a locked entry recorded — as declared, so // the constraint and the transport (an SSH address, say) are the package's // own — and an offline run walks the closure from the lock alone. @@ -528,7 +552,13 @@ func (r *resolver) resolveRemoteArchive(o *origin.Origin, req request) (*lock.En id := o.Identity() old := r.locked(id, req.constraint) if e, ok, err := r.reuseLocked(old); ok || err != nil { - return e, err + if err == nil { + err = r.verifyLocked(o, old) + } + if err != nil { + return nil, fmt.Errorf("%s: usage %q: %v", req.requester, req.resource, err) + } + return e, nil } tmp, err := os.MkdirTemp("", "syspi-dl-*") if err != nil { @@ -539,6 +569,11 @@ func (r *resolver) resolveRemoteArchive(o *origin.Origin, req request) (*lock.En if err := r.download(o.Source, file); err != nil { return nil, fmt.Errorf("%s: usage %q: %v", req.requester, req.resource, err) } + // The declared checksums are checked before the archive is opened, so no + // member of a tampered download is read. + if err := digest.Verify(file, o.Checksums); err != nil { + return nil, fmt.Errorf("%s: usage %q: %v", req.requester, req.resource, err) + } a, err := project.OpenArchive(file) if err != nil { return nil, fmt.Errorf("%s: usage %q: %v", req.requester, req.resource, err) @@ -547,6 +582,9 @@ func (r *resolver) resolveRemoteArchive(o *origin.Origin, req request) (*lock.En if old != nil && old.SHA256 != a.SHA256 { return nil, fmt.Errorf("%s: usage %q: downloaded archive SHA-256 %s does not match the lock's %s", req.requester, req.resource, a.SHA256, old.SHA256) } + if o.Hosted() && a.Project.Name != o.Name { + return nil, fmt.Errorf("%s: usage %q: the purl names %q but the archive's %s names %q", req.requester, req.resource, o.Name, project.ProjectFile, a.Project.Name) + } v, _ := version.Parse(a.Project.Version) if !req.constraint.Check(v) { return nil, fmt.Errorf("%s: usage %q wants %s but the archive holds %s %s", req.requester, req.resource, req.constraint, a.Project.Name, a.Project.Version) diff --git a/internal/install/install_test.go b/internal/install/install_test.go index 175e807..3462ce0 100644 --- a/internal/install/install_test.go +++ b/internal/install/install_test.go @@ -5,6 +5,8 @@ package install import ( "bytes" "context" + "crypto/sha1" + "encoding/hex" "net/http" "net/http/httptest" "os" @@ -586,3 +588,94 @@ func TestPublishRollsBackWhenTheLockCannotBeWritten(t *testing.T) { t.Fatalf("lib/ must match the lock still on disk, got %v", names) } } + +// A .kpar named by a download_url purl installs as the plain IRI does, after +// the declared checksum is verified; a mismatch installs nothing. +func TestHostedArchive(t *testing.T) { + pkg := t.TempDir() + write(t, pkg, ".project.json", `{"name":"tools","publisher":"acme","version":"3.1.0"}`) + write(t, pkg, "Tools.sysml", "package Tools;\n") + kpar := filepath.Join(t.TempDir(), "release.kpar") + if _, err := project.Bundle(pkg, kpar, time.Unix(0, 0)); err != nil { + t.Fatal(err) + } + sum, _ := digest.File(kpar) + srv := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir(kpar)))) + defer srv.Close() + enc := origin.Encode(srv.URL + "/release.kpar") + id := "pkg:generic/tools?download_url=" + enc + good := id + "&checksum=sha256:" + sum + wrong := id + "&checksum=sha256:" + strings.Repeat("00", 32) + + // The wrong checksum: an error before the archive is opened, nothing under lib/, no lock. + bad := consumer(t, project.Usage{Resource: wrong}) + before := treeList(t, bad) + runErr(t, bad, Options{}, "checksum mismatch: the resource declares sha256 "+strings.Repeat("00", 32)+", the archive has "+sum) + if after := treeList(t, bad); strings.Join(after, "\n") != strings.Join(before, "\n") { + t.Fatalf("a failed checksum changed the tree:\n%v\n%v", before, after) + } + + dir := consumer(t, project.Usage{Resource: good, VersionConstraint: "^3.0.0"}) + res, _ := run(t, dir, Options{}) + if len(res.Installed) != 1 || res.Installed[0] != id { + t.Fatalf("result %+v", res) + } + if d, _ := digest.File(filepath.Join(dir, "lib", "acme-tools_3.1.0.kpar")); d != sum { + t.Fatal("installed archive differs") + } + lk, _, _ := lock.Read(dir, project.LockFile) + e := lk.Find(id) + if e == nil || e.Kind != lock.KindArchive || e.SHA256 != sum || e.Source != srv.URL+"/release.kpar" || e.Name != "tools" || e.Publisher != "acme" || e.Version != "3.1.0" { + t.Fatalf("lock entry %+v", e) + } + // Offline from the lock with the server gone; a second declared algorithm is + // checked against the installed archive. + res2, _ := run(t, dir, Options{Offline: true}) + if len(res2.Reused) != 1 { + t.Fatalf("offline %+v", res2) + } + sha1sum := digestOf(t, kpar, "sha1") + rewrite(t, dir, project.Usage{Resource: id + "&checksum=sha1:" + sha1sum + "%2Csha256:" + sum}) + if res3, _ := run(t, dir, Options{Offline: true}); len(res3.Reused) != 1 { + t.Fatalf("offline with two checksums %+v", res3) + } + rewrite(t, dir, project.Usage{Resource: wrong}) + runErr(t, dir, Options{Offline: true}, "the lock records "+sum) + rewrite(t, dir, project.Usage{Resource: id + "&checksum=sha1:" + strings.Repeat("00", 20)}) + runErr(t, dir, Options{Offline: true}, "checksum mismatch: the resource declares sha1 "+strings.Repeat("00", 20)) + + // Without a checksum the download is trusted as a plain IRI is; the lock still pins its digest. + plain := consumer(t, project.Usage{Resource: id}) + if res, _ := run(t, plain, Options{}); len(res.Installed) != 1 { + t.Fatalf("result %+v", res) + } + // The version is the archive's own and is held to @ and versionConstraint. + runErr(t, consumer(t, project.Usage{Resource: "pkg:generic/tools@3.0.0?download_url=" + enc}), Options{}, "wants 3.0.0 but the archive holds tools 3.1.0") + runErr(t, consumer(t, project.Usage{Resource: id, VersionConstraint: "^4.0.0"}), Options{}, "wants ^4.0.0 but the archive holds tools 3.1.0") + // The purl name is the archive's name. + runErr(t, consumer(t, project.Usage{Resource: "pkg:generic/widgets?download_url=" + enc}), Options{}, `the purl names "widgets" but the archive's .project.json names "tools"`) + // Offline without the archive installed refuses as the plain IRI does. + runErr(t, consumer(t, project.Usage{Resource: good}), Options{Offline: true}, "offline: "+srv.URL+"/release.kpar is not installed") +} + +// rewrite replaces the consumer's usages. +func rewrite(t *testing.T, dir string, usages ...project.Usage) { + t.Helper() + p := &project.Project{Name: "app", Publisher: "test", Version: "0.1.0", Usage: usages} + if err := p.Write(dir); err != nil { + t.Fatal(err) + } +} + +// digestOf is the hex digest of a file under the named checksum algorithm. +func digestOf(t *testing.T, path, algo string) string { + t.Helper() + data, _ := os.ReadFile(path) + switch algo { + case "sha1": + sum := sha1.Sum(data) + return hex.EncodeToString(sum[:]) + } + t.Fatalf("unknown algorithm %s", algo) + return "" +} diff --git a/internal/origin/origin.go b/internal/origin/origin.go index 49c44e2..7969b00 100644 --- a/internal/origin/origin.go +++ b/internal/origin/origin.go @@ -21,6 +21,7 @@ import ( "runtime" "strings" + "github.com/Open-MBEE/SysPI/internal/digest" "github.com/Open-MBEE/SysPI/internal/version" ) @@ -75,6 +76,10 @@ const Spellings = "pkg:generic/?vcs_url=%2B:%2F%2F%2 "pkg:git//[@][#], or +:///[@][#] " + "(vcs: git, svn, hg; transports per ECMA-427 Annex B.3.5, plus file for a repository on this machine)" +// ArchiveSpelling is the accepted way to write a .kpar hosted at an http(s) URL. +const ArchiveSpelling = "pkg:generic/[@]?download_url=https:%2F%2F%2F[&checksum=:] " + + "(ECMA-427 Annex B.3.5; checksum optional, items comma-separated)" + // Origin is a parsed resource. type Origin struct { Kind Kind @@ -94,6 +99,8 @@ type Origin struct { // Source is what the VCS executable is given, or the archive path/URL, or // the directory for LocalDir, or the IRI for Standard. Source string + // Checksums are the checksum qualifier's items for a hosted .kpar, nil for none. + Checksums []digest.Checksum // Resource is the resource exactly as written. Resource string } @@ -120,14 +127,21 @@ func (o *Origin) VCSURL(withVersion bool) string { } // Identity is the canonical, version-free identity of the package: -// pkg:generic/?vcs_url=… for a repository, otherwise the source. +// pkg:generic/?vcs_url=… for a repository, pkg:generic/?download_url=… +// for a hosted .kpar, otherwise the source. func (o *Origin) Identity() string { - if o.Kind != VCS { - return o.Source + switch { + case o.Kind == VCS: + return "pkg:generic/" + Encode(o.Name) + "?vcs_url=" + Encode(o.VCSURL(false)) + case o.Hosted(): + return "pkg:generic/" + Encode(o.Name) + "?download_url=" + Encode(o.Source) } - return "pkg:generic/" + Encode(o.Name) + "?vcs_url=" + Encode(o.VCSURL(false)) + return o.Source } +// Hosted reports whether the resource is a .kpar written as a download_url purl. +func (o *Origin) Hosted() bool { return o.Kind == RemoteArchive && o.Name != "" } + // Key is the identity with a .git suffix dropped from the repository path: // what a lock entry and a usage are matched by, however the suffix was spelled. func (o *Origin) Key() string { @@ -140,12 +154,24 @@ func (o *Origin) Key() string { } // Canonical is the identity with the exact version inside vcs_url when one -// was named. +// was named; for a hosted .kpar, the purl with its version and, first in +// qualifier order, its checksums. func (o *Origin) Canonical() string { - if o.Kind != VCS { - return o.Source + switch { + case o.Kind == VCS: + return "pkg:generic/" + Encode(o.Name) + "?vcs_url=" + Encode(o.VCSURL(true)) + case o.Hosted(): + s := "pkg:generic/" + Encode(o.Name) + if o.Version != "" { + s += "@" + Encode(o.Version) + } + s += "?" + if len(o.Checksums) > 0 { + s += "checksum=" + Encode(digest.FormatChecksums(o.Checksums)) + "&" + } + return s + "download_url=" + Encode(o.Source) } - return "pkg:generic/" + Encode(o.Name) + "?vcs_url=" + Encode(o.VCSURL(true)) + return o.Source } // Encode percent-encodes a purl component the way the canonical purl form @@ -222,43 +248,58 @@ func parsePkg(res, rest string) (*Origin, error) { } } -// parseGeneric handles pkg:generic/[@]?vcs_url=. +// parseGeneric handles pkg:generic/[@] with a vcs_url +// qualifier (a repository) or a download_url one (a hosted .kpar). func parseGeneric(res, tail string) (*Origin, error) { namePart, query, hasQuery := strings.Cut(tail, "?") if !hasQuery { - return nil, fmt.Errorf("resource %q: pkg:generic needs a vcs_url qualifier; write %s", res, Spellings) + return nil, fmt.Errorf("resource %q: pkg:generic needs a vcs_url or download_url qualifier; write %s or %s", res, Spellings, ArchiveSpelling) } q, err := url.ParseQuery(strings.ReplaceAll(query, "+", "%2B")) if err != nil { return nil, fmt.Errorf("resource %q: invalid qualifiers: %v", res, err) } - for k := range q { - if k != "vcs_url" { - return nil, fmt.Errorf("resource %q: qualifier %q is not supported; only vcs_url is", res, k) + for k, vs := range q { + if k != "vcs_url" && k != "download_url" && k != "checksum" { + return nil, fmt.Errorf("resource %q: qualifier %q is not supported; vcs_url, download_url and checksum are", res, k) + } + if len(vs) != 1 || vs[0] == "" { + return nil, fmt.Errorf("resource %q: pkg:generic needs exactly one non-empty %s qualifier", res, k) } - } - if len(q["vcs_url"]) != 1 || q["vcs_url"][0] == "" { - return nil, fmt.Errorf("resource %q: pkg:generic needs exactly one vcs_url qualifier", res) - } - o, err := parseVCSURL(res, q["vcs_url"][0]) - if err != nil { - return nil, err } if strings.Contains(namePart, "/") { - return nil, fmt.Errorf("resource %q: pkg:generic takes no namespace; the name is the repository's last path segment", res) + return nil, fmt.Errorf("resource %q: pkg:generic takes no namespace; write the name alone", res) } name, ver, _ := strings.Cut(namePart, "@") name, err = url.PathUnescape(name) if err != nil || name == "" { return nil, fmt.Errorf("resource %q: invalid purl name", res) } - if name != o.Name { - return nil, fmt.Errorf("resource %q: purl name %q must be the repository's last path segment %q", res, name, o.Name) - } if ver != "" { if _, err := version.Parse(ver); err != nil { return nil, fmt.Errorf("resource %q: %q is not a semantic version", res, ver) } + } + _, hasVCS := q["vcs_url"] + _, hasDownload := q["download_url"] + switch { + case hasVCS && hasDownload: + return nil, fmt.Errorf("resource %q: vcs_url and download_url exclude each other; a package is either a repository or a hosted .kpar", res) + case hasDownload: + return parseDownload(res, name, ver, q) + case !hasVCS: + return nil, fmt.Errorf("resource %q: a checksum qualifier needs a download_url to verify; write %s", res, ArchiveSpelling) + case len(q) > 1: + return nil, fmt.Errorf("resource %q: a checksum qualifier is only verified with download_url; a repository is locked by its tree digest", res) + } + o, err := parseVCSURL(res, q["vcs_url"][0]) + if err != nil { + return nil, err + } + if name != o.Name { + return nil, fmt.Errorf("resource %q: purl name %q must be the repository's last path segment %q", res, name, o.Name) + } + if ver != "" { if o.Version != "" && o.Version != ver { return nil, fmt.Errorf("resource %q: version %s in the purl and %s in vcs_url disagree", res, ver, o.Version) } @@ -268,6 +309,31 @@ func parseGeneric(res, tail string) (*Origin, error) { return o, nil } +// parseDownload handles the download_url form: the decoded qualifier is the +// http(s) URL the .kpar is fetched from, checked as a plain .kpar IRI is but +// for the extension, which a hosted archive's URL need not spell. +func parseDownload(res, name, ver string, q url.Values) (*Origin, error) { + raw := q["download_url"][0] + u, err := url.Parse(raw) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return nil, fmt.Errorf("resource %q: download_url %q is not an http(s) URL", res, raw) + } + if u.User != nil { + return nil, fmt.Errorf("resource %q: credentials in download_url are not supported; add a netrc entry for %s instead and keep the URL bare", res, u.Hostname()) + } + if u.Fragment != "" || strings.HasSuffix(raw, "#") { + return nil, fmt.Errorf("resource %q: a .kpar cannot have a #", res) + } + o := &Origin{Kind: RemoteArchive, Name: name, Version: ver, Source: raw, Resource: res} + if cs, ok := q["checksum"]; ok { + o.Checksums, err = digest.ParseChecksums(cs[0]) + if err != nil { + return nil, fmt.Errorf("resource %q: %v", res, err) + } + } + return o, nil +} + // parseGitAlias handles pkg:git//[@][#] as // git+https:///. func parseGitAlias(res, tail string) (*Origin, error) { diff --git a/internal/origin/origin_test.go b/internal/origin/origin_test.go index a1842f0..86a4ab1 100644 --- a/internal/origin/origin_test.go +++ b/internal/origin/origin_test.go @@ -89,7 +89,7 @@ func TestRefused(t *testing.T) { {"hg://hg.example.org/repo", "hg+hg is not a vcs_url scheme for hg (ECMA-427 Annex B.3.5 allows hg+http, hg+https, hg+static-http, hg+ssh, hg+file)"}, {"git+ftp://host/repo", "git+ftp is not a vcs_url scheme for git"}, {"bzr+https://host/repo", "version-control system \"bzr\" is not supported; supported: git, svn, hg"}, - {"pkg:generic/widgets", "pkg:generic needs a vcs_url qualifier"}, + {"pkg:generic/widgets", "pkg:generic needs a vcs_url or download_url qualifier"}, {"pkg:generic/other?vcs_url=git%2Bhttps:%2F%2Fgithub.com%2Facme%2Fwidgets", "purl name \"other\" must be the repository's last path segment \"widgets\""}, {"pkg:generic/widgets?vcs_url=git%2Bhttps:%2F%2Fgithub.com%2Facme%2Fwidgets&arch=x", "qualifier \"arch\" is not supported"}, {"pkg:generic/widgets@1.0.0?vcs_url=git%2Bhttps:%2F%2Fgithub.com%2Facme%2Fwidgets%402.0.0", "version 1.0.0 in the purl and 2.0.0 in vcs_url disagree"}, @@ -173,3 +173,73 @@ func TestDriveLetterFileURIs(t *testing.T) { t.Errorf("file:///C:/tools.kpar: %v %+v", err, o) } } + +const hostedURL = "https://dist.example.org/releases/v3.0.0/acme-tools.kpar?format=kpar" + +// A hosted .kpar is a download_url purl: version-free identity, canonical +// form with the version and the sorted checksums, the URL as the source. +func TestHostedArchive(t *testing.T) { + sha := strings.Repeat("ab", 32) + enc := "https:%2F%2Fdist.example.org%2Freleases%2Fv3.0.0%2Facme-tools.kpar%3Fformat%3Dkpar" + for _, tc := range []struct{ in, canonical, identity, version string }{ + {"pkg:generic/tools?download_url=" + enc, "pkg:generic/tools?download_url=" + enc, "pkg:generic/tools?download_url=" + enc, ""}, + {"pkg:generic/tools?download_url=https%3A%2F%2Fdist.example.org%2Freleases%2Fv3.0.0%2Facme-tools.kpar%3Fformat%3Dkpar", "pkg:generic/tools?download_url=" + enc, "pkg:generic/tools?download_url=" + enc, ""}, + {"pkg:generic/tools@3.0.0?download_url=" + enc + "&checksum=SHA256:" + strings.ToUpper(sha), "pkg:generic/tools@3.0.0?checksum=sha256:" + sha + "&download_url=" + enc, "pkg:generic/tools?download_url=" + enc, "3.0.0"}, + {"pkg:generic/tools?checksum=sha256:" + sha + "%2Cmd5:" + strings.Repeat("01", 16) + "&download_url=" + enc, "pkg:generic/tools?checksum=md5:" + strings.Repeat("01", 16) + "%2Csha256:" + sha + "&download_url=" + enc, "pkg:generic/tools?download_url=" + enc, ""}, + {"pkg:generic/Tools%20Model?download_url=http:%2F%2F127.0.0.1:8000%2Fexport", "pkg:generic/Tools%20Model?download_url=http:%2F%2F127.0.0.1:8000%2Fexport", "pkg:generic/Tools%20Model?download_url=http:%2F%2F127.0.0.1:8000%2Fexport", ""}, + } { + o, err := Parse(tc.in) + if err != nil { + t.Errorf("%s: %v", tc.in, err) + continue + } + if o.Kind != RemoteArchive || !o.Hosted() || o.Canonical() != tc.canonical || o.Identity() != tc.identity || o.Key() != tc.identity || o.Version != tc.version { + t.Errorf("%s:\n canonical %s\n want %s\n identity %s version %q kind %s", tc.in, o.Canonical(), tc.canonical, o.Identity(), o.Version, o.Kind) + continue + } + if strings.HasPrefix(o.Source, "https://") && o.Source != hostedURL { + t.Errorf("%s: source %s", tc.in, o.Source) + } + again, err := Parse(o.Canonical()) + if err != nil || again.Canonical() != o.Canonical() || again.Identity() != o.Identity() || again.Source != o.Source || again.Version != o.Version || len(again.Checksums) != len(o.Checksums) { + t.Errorf("%s: canonical form does not round-trip: %v %+v", tc.in, err, again) + } + } + o, _ := Parse("pkg:generic/tools?download_url=" + enc + "&checksum=sha256:" + sha) + if len(o.Checksums) != 1 || o.Checksums[0].Algorithm != "sha256" || o.Checksums[0].Value != sha { + t.Errorf("checksums %+v", o.Checksums) + } + plain, _ := Parse(hostedURL[:strings.Index(hostedURL, "?")]) + if plain.Hosted() || plain.Identity() != plain.Source || len(plain.Checksums) != 0 { + t.Errorf("a plain .kpar IRI is not a hosted purl: %+v", plain) + } +} + +func TestHostedArchiveRefusals(t *testing.T) { + sha := strings.Repeat("ab", 32) + enc := "https:%2F%2Fdist.example.org%2Facme-tools.kpar" + for _, tc := range []struct{ in, want string }{ + {"pkg:generic/tools?download_url=" + enc + "&vcs_url=git%2Bhttps:%2F%2Fgithub.com%2Facme%2Ftools", "vcs_url and download_url exclude each other"}, + {"pkg:generic/tools?checksum=sha256:" + sha, "a checksum qualifier needs a download_url"}, + {"pkg:generic/tools?vcs_url=git%2Bhttps:%2F%2Fgithub.com%2Facme%2Ftools&checksum=sha256:" + sha, "only verified with download_url"}, + {"pkg:generic/tools?download_url=" + enc + "&download_url=" + enc, "exactly one non-empty download_url qualifier"}, + {"pkg:generic/tools?download_url=", "exactly one non-empty download_url qualifier"}, + {"pkg:generic/tools?download_url=" + enc + "&checksum=", "exactly one non-empty checksum qualifier"}, + {"pkg:generic/tools?download_url=" + enc + "&checksum=sha256:" + sha + "&checksum=md5:" + strings.Repeat("01", 16), "exactly one non-empty checksum qualifier"}, + {"pkg:generic/tools?download_url=" + enc + "&checksum=crc32:deadbeef", `checksum algorithm "crc32" is not supported`}, + {"pkg:generic/tools?download_url=" + enc + "&checksum=sha256:abcd", "a sha256 value is 64 hex digits"}, + {"pkg:generic/tools?download_url=" + enc + "&checksum=" + sha, "is not :"}, + {"pkg:generic/tools?download_url=ftp:%2F%2Fdist.example.org%2Ftools.kpar", "is not an http(s) URL"}, + {"pkg:generic/tools?download_url=dist.example.org%2Ftools.kpar", "is not an http(s) URL"}, + {"pkg:generic/tools?download_url=https:%2F%2Fuser:pw%40dist.example.org%2Ftools.kpar", "credentials in download_url are not supported"}, + {"pkg:generic/tools?download_url=" + enc + "%23sub", "a .kpar cannot have a #"}, + {"pkg:generic/tools?download_url=" + enc + "#sub", "a .kpar cannot have a #"}, + {"pkg:generic/acme/tools?download_url=" + enc, "takes no namespace"}, + {"pkg:generic/tools@three?download_url=" + enc, `"three" is not a semantic version`}, + {"pkg:generic/tools?download_url=" + enc + "&arch=x", `qualifier "arch" is not supported; vcs_url, download_url and checksum are`}, + } { + if _, err := Parse(tc.in); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: got %v, want %q", tc.in, err, tc.want) + } + } +} diff --git a/internal/project/meta.go b/internal/project/meta.go index d02c63b..9b6a790 100644 --- a/internal/project/meta.go +++ b/internal/project/meta.go @@ -103,8 +103,12 @@ func ReadMeta(dir string) (m *Meta, ok bool, err error) { // Marshal renders the metadata as indented JSON with sorted object keys and // a trailing newline. func (m *Meta) Marshal() []byte { - data, _ := json.MarshalIndent(m, "", " ") - return append(data, '\n') + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + enc.Encode(m) + return buf.Bytes() } // Write writes /.meta.json. diff --git a/internal/project/project.go b/internal/project/project.go index c134bd9..996c4ae 100644 --- a/internal/project/project.go +++ b/internal/project/project.go @@ -109,8 +109,12 @@ var ErrNoProject = errors.New("no " + ProjectFile) // Marshal renders the project as indented JSON with a trailing newline, // properties in Table 12 order. func (p *Project) Marshal() []byte { - data, _ := json.MarshalIndent(p, "", " ") - return append(data, '\n') + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + enc.Encode(p) + return buf.Bytes() } // Write writes /.project.json. diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 26a0cee..1fadd5b 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -8,6 +8,8 @@ package e2e import ( "bytes" + "crypto/sha256" + "encoding/hex" "net/http" "net/http/httptest" "os" @@ -426,3 +428,62 @@ func TestRefusals(t *testing.T) { t.Fatalf("missing git: %v\n%s", err, out) } } + +// A .kpar served over HTTP and named by a download_url purl: a wrong checksum +// installs nothing and leaves .project.json alone; the right one installs it. +func TestHostedArchive(t *testing.T) { + bin := syspiBinary(t) + root := t.TempDir() + tools := filepath.Join(root, "tools") + write(t, tools, ".project.json", "{\n \"name\": \"tools\",\n \"publisher\": \"acme\",\n \"version\": \"3.0.0\"\n}\n") + write(t, tools, "Tools.sysml", "package Tools {\n part def Tool;\n}\n") + served := filepath.Join(root, "served") + os.MkdirAll(served, 0o755) + kpar := filepath.Join(served, "release.kpar") + if out, err := shErr(root, []string{"SOURCE_DATE_EPOCH=1700000000"}, bin, "bundle", "-o", kpar, tools); err != nil { + t.Fatalf("bundle: %v\n%s", err, out) + } + data, _ := os.ReadFile(kpar) + sum := sha256.Sum256(data) + hexSum := hex.EncodeToString(sum[:]) + srv := httptest.NewServer(http.FileServer(http.Dir(served))) + defer srv.Close() + enc := strings.NewReplacer("/", "%2F").Replace(srv.URL) + "%2Frelease.kpar" + id := "pkg:generic/tools?download_url=" + enc + + app := filepath.Join(root, "app") + os.MkdirAll(app, 0o755) + sh(t, app, bin, "init", "-publisher", "test", "-name", "app") + pj, _ := os.ReadFile(filepath.Join(app, ".project.json")) + out, err := shErr(app, nil, bin, "add", id+"&checksum=sha256:"+strings.Repeat("00", 32)) + if err == nil || !strings.Contains(out, "checksum mismatch: the resource declares sha256 "+strings.Repeat("00", 32)+", the archive has "+hexSum) { + t.Fatalf("wrong checksum: %v\n%s", err, out) + } + if after, _ := os.ReadFile(filepath.Join(app, ".project.json")); !bytes.Equal(pj, after) { + t.Fatalf(".project.json changed by a failed add:\n%s", after) + } + for _, p := range []string{"lib", ".project.lock.json"} { + if _, err := os.Stat(filepath.Join(app, p)); err == nil { + t.Fatalf("%s exists after a failed add", p) + } + } + sh(t, app, bin, "add", "pkg:generic/tools@3.0.0?download_url="+enc+"&checksum=SHA256:"+strings.ToUpper(hexSum)) + pj, _ = os.ReadFile(filepath.Join(app, ".project.json")) + if want := `"resource": "pkg:generic/tools@3.0.0?checksum=sha256:` + hexSum + `&download_url=` + enc + `"`; !strings.Contains(string(pj), want) { + t.Fatalf(".project.json lacks the canonical form %s:\n%s", want, pj) + } + if _, err := os.Stat(filepath.Join(app, "lib", "acme-tools_3.0.0.kpar")); err != nil { + t.Fatal(err) + } + lk, _ := os.ReadFile(filepath.Join(app, ".project.lock.json")) + if lockField(t, string(lk), id, "kind") != "archive" || lockField(t, string(lk), id, "sha256") != hexSum || lockField(t, string(lk), id, "source") != srv.URL+"/release.kpar" { + t.Fatalf("lock entry:\n%s", lk) + } + // Offline from lib/ and the lock with the server gone. + srv.Close() + sh(t, app, bin, "install", "-offline") + sh(t, app, bin, "lock", "-offline") + if lk2, _ := os.ReadFile(filepath.Join(app, ".project.lock.json")); !bytes.Equal(lk, lk2) { + t.Fatalf("lock changed offline:\n%s\n%s", lk, lk2) + } +} From a772c5e2510a57a9581258652a3d822229293793 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 06:29:42 +0000 Subject: [PATCH 2/4] fix(origin): decode the percent-encoded version of a pkg:generic purl The canonical form writes @1.2.3%2Bci; parsing it validated the encoded text and refused the build metadata. Co-Authored-By: jason.han --- internal/origin/origin.go | 3 +++ internal/origin/origin_test.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/internal/origin/origin.go b/internal/origin/origin.go index 7969b00..6de8151 100644 --- a/internal/origin/origin.go +++ b/internal/origin/origin.go @@ -276,6 +276,9 @@ func parseGeneric(res, tail string) (*Origin, error) { return nil, fmt.Errorf("resource %q: invalid purl name", res) } if ver != "" { + if ver, err = url.PathUnescape(ver); err != nil { + return nil, fmt.Errorf("resource %q: invalid purl version", res) + } if _, err := version.Parse(ver); err != nil { return nil, fmt.Errorf("resource %q: %q is not a semantic version", res, ver) } diff --git a/internal/origin/origin_test.go b/internal/origin/origin_test.go index 86a4ab1..3615501 100644 --- a/internal/origin/origin_test.go +++ b/internal/origin/origin_test.go @@ -186,6 +186,8 @@ func TestHostedArchive(t *testing.T) { {"pkg:generic/tools?download_url=https%3A%2F%2Fdist.example.org%2Freleases%2Fv3.0.0%2Facme-tools.kpar%3Fformat%3Dkpar", "pkg:generic/tools?download_url=" + enc, "pkg:generic/tools?download_url=" + enc, ""}, {"pkg:generic/tools@3.0.0?download_url=" + enc + "&checksum=SHA256:" + strings.ToUpper(sha), "pkg:generic/tools@3.0.0?checksum=sha256:" + sha + "&download_url=" + enc, "pkg:generic/tools?download_url=" + enc, "3.0.0"}, {"pkg:generic/tools?checksum=sha256:" + sha + "%2Cmd5:" + strings.Repeat("01", 16) + "&download_url=" + enc, "pkg:generic/tools?checksum=md5:" + strings.Repeat("01", 16) + "%2Csha256:" + sha + "&download_url=" + enc, "pkg:generic/tools?download_url=" + enc, ""}, + {"pkg:generic/tools@1.2.3%2Bci.7?download_url=" + enc, "pkg:generic/tools@1.2.3%2Bci.7?download_url=" + enc, "pkg:generic/tools?download_url=" + enc, "1.2.3+ci.7"}, + {"pkg:generic/tools@1.2.3+ci.7?download_url=" + enc, "pkg:generic/tools@1.2.3%2Bci.7?download_url=" + enc, "pkg:generic/tools?download_url=" + enc, "1.2.3+ci.7"}, {"pkg:generic/Tools%20Model?download_url=http:%2F%2F127.0.0.1:8000%2Fexport", "pkg:generic/Tools%20Model?download_url=http:%2F%2F127.0.0.1:8000%2Fexport", "pkg:generic/Tools%20Model?download_url=http:%2F%2F127.0.0.1:8000%2Fexport", ""}, } { o, err := Parse(tc.in) From c862288a7974113c558faef11c54696b36a87027 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 06:29:42 +0000 Subject: [PATCH 3/4] fix(install): hold every usage of a hosted archive to its declared checksums A second usage of an archive already resolved in the run, and a locked entry reused without its installed copy, were not checked against the checksums they declare. The resolver now keeps the digests known for an archive in the run, compares each usage to them, and downloads an archive a lock-only run cannot otherwise check. Co-Authored-By: jason.han --- internal/digest/checksum.go | 48 ++++++++++---- internal/install/install.go | 106 ++++++++++++++++++++++++++----- internal/install/install_test.go | 28 +++++++- 3 files changed, 151 insertions(+), 31 deletions(-) diff --git a/internal/digest/checksum.go b/internal/digest/checksum.go index c199f29..4819dde 100644 --- a/internal/digest/checksum.go +++ b/internal/digest/checksum.go @@ -95,29 +95,51 @@ func FormatChecksums(sums []Checksum) string { return strings.Join(parts, ",") } -// Verify checks the file's bytes against every declared checksum in one pass -// and reports the first that does not match. -func Verify(path string, sums []Checksum) error { +// Sums computes the file's digest under each item's algorithm in one pass, +// keyed by algorithm. +func Sums(path string, sums []Checksum) (map[string]string, error) { + out := map[string]string{} if len(sums) == 0 { - return nil + return out, nil } f, err := os.Open(path) if err != nil { - return err + return nil, err } defer f.Close() - hashes := make([]hash.Hash, len(sums)) - writers := make([]io.Writer, len(sums)) - for i, c := range sums { - hashes[i] = algorithms[c.Algorithm]() - writers[i] = hashes[i] + hashes := map[string]hash.Hash{} + var writers []io.Writer + for _, c := range sums { + if _, dup := hashes[c.Algorithm]; !dup { + hashes[c.Algorithm] = algorithms[c.Algorithm]() + writers = append(writers, hashes[c.Algorithm]) + } } if _, err := io.Copy(io.MultiWriter(writers...), f); err != nil { + return nil, err + } + for algo, h := range hashes { + out[algo] = hex.EncodeToString(h.Sum(nil)) + } + return out, nil +} + +// Verify checks the file's bytes against every declared checksum and reports +// the first that does not match. +func Verify(path string, sums []Checksum) error { + got, err := Sums(path, sums) + if err != nil { return err } - for i, c := range sums { - if got := hex.EncodeToString(hashes[i].Sum(nil)); got != c.Value { - return fmt.Errorf("checksum mismatch: the resource declares %s %s, the archive has %s", c.Algorithm, c.Value, got) + return Compare(sums, got) +} + +// Compare holds the declared checksums to digests computed from the archive, +// keyed by algorithm; an algorithm absent from got is not checked. +func Compare(sums []Checksum, got map[string]string) error { + for _, c := range sums { + if g, ok := got[c.Algorithm]; ok && g != c.Value { + return fmt.Errorf("checksum mismatch: the resource declares %s %s, the archive has %s", c.Algorithm, c.Value, g) } } return nil diff --git a/internal/install/install.go b/internal/install/install.go index 44cfc33..1af94b0 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -62,6 +62,10 @@ type resolver struct { old *lock.Lock fresh *lock.Lock chosen map[string]*lock.Entry // identity -> entry resolved in this run + // sums holds, per archive identity, the digests of its bytes known in + // this run (the lock's SHA-256, the checksums verified on download or + // against the installed copy), so a second usage is held to them too. + sums map[string]map[string]string queue []request res *Result libDir string @@ -117,7 +121,7 @@ func Run(ctx context.Context, opts Options) (*Result, error) { old = lock.New() } keyResources(old) - r := &resolver{ctx: ctx, opts: opts, proj: proj, old: old, fresh: lock.New(), chosen: map[string]*lock.Entry{}, res: &Result{}, libDir: filepath.Join(root, project.LibDir), slots: map[string]slot{}} + r := &resolver{ctx: ctx, opts: opts, proj: proj, old: old, fresh: lock.New(), chosen: map[string]*lock.Entry{}, sums: map[string]map[string]string{}, res: &Result{}, libDir: filepath.Join(root, project.LibDir), slots: map[string]slot{}} for _, u := range proj.Usage { c, err := usageConstraint(u) if err != nil { @@ -271,6 +275,15 @@ func (r *resolver) resolve(req request) error { return fmt.Errorf("%s: usage %q wants %s but %s is already resolved to %s (required by another usage)", req.requester, req.resource, req.constraint, id, e.Version) } } + if e.Kind == lock.KindArchive { + checked, err := r.verifyDeclared(o, e) + if err == nil && !checked { + err = fmt.Errorf("the declared %s checksum cannot be verified: the archive is not installed", unverified(o, r.sums[id])) + } + if err != nil { + return fmt.Errorf("%s: usage %q: %v", req.requester, req.resource, err) + } + } return nil } var e *lock.Entry @@ -456,23 +469,73 @@ func (r *resolver) reuseLocked(old *lock.Entry) (*lock.Entry, bool, error) { return nil, false, nil } -// verifyLocked holds a reused lock entry to the checksums a hosted .kpar's -// resource declares: a sha256 item against the lock's digest, every item -// against the installed archive when it is present. -func (r *resolver) verifyLocked(o *origin.Origin, old *lock.Entry) error { +// verifyDeclared holds the archive an entry records to the checksums a +// hosted usage declares: against the digests known this run, else against +// the installed or staged copy. It reports whether every item was checked. +func (r *resolver) verifyDeclared(o *origin.Origin, e *lock.Entry) (bool, error) { if len(o.Checksums) == 0 { - return nil + return true, nil } + known := r.known(o.Identity()) + known["sha256"] = e.SHA256 + if err := digest.Compare(o.Checksums, known); err != nil { + return false, err + } + var unchecked []digest.Checksum for _, c := range o.Checksums { - if c.Algorithm == "sha256" && c.Value != old.SHA256 { - return fmt.Errorf("checksum mismatch: the resource declares sha256 %s, the lock records %s; remove the lock entry to resolve it again", c.Value, old.SHA256) + if _, ok := known[c.Algorithm]; !ok { + unchecked = append(unchecked, c) } } - dst := r.lockedArchivePath(old) - if _, err := os.Stat(dst); err != nil { - return nil + if len(unchecked) == 0 { + return true, nil + } + file := r.archiveFile(e) + if file == "" { + return false, nil + } + got, err := digest.Sums(file, unchecked) + if err != nil { + return false, err + } + for algo, v := range got { + known[algo] = v + } + return true, digest.Compare(unchecked, got) +} + +// known is the digest map of an archive identity, created on first use. +func (r *resolver) known(id string) map[string]string { + if r.sums[id] == nil { + r.sums[id] = map[string]string{} + } + return r.sums[id] +} + +// unverified names the declared algorithms no digest is known for. +func unverified(o *origin.Origin, known map[string]string) string { + var out []string + for _, c := range o.Checksums { + if _, ok := known[c.Algorithm]; !ok { + out = append(out, c.Algorithm) + } + } + return strings.Join(out, ", ") +} + +// archiveFile is the staged or installed copy of a locked archive, "" when +// there is none to read. +func (r *resolver) archiveFile(e *lock.Entry) string { + dst := r.lockedArchivePath(e) + for _, it := range r.items { + if it.dst == dst { + return it.src + } + } + if _, err := os.Stat(dst); err == nil { + return dst } - return digest.Verify(dst, o.Checksums) + return "" } // lockedArchivePath is where a locked archive entry is installed under lib/. @@ -551,14 +614,18 @@ func (r *resolver) download(url string, dst string) error { func (r *resolver) resolveRemoteArchive(o *origin.Origin, req request) (*lock.Entry, error) { id := o.Identity() old := r.locked(id, req.constraint) - if e, ok, err := r.reuseLocked(old); ok || err != nil { - if err == nil { - err = r.verifyLocked(o, old) - } + // A locked entry is reused only when every declared checksum was checked + // against it or the installed copy; otherwise the download checks them. + if old != nil { + checked, err := r.verifyDeclared(o, old) if err != nil { return nil, fmt.Errorf("%s: usage %q: %v", req.requester, req.resource, err) } - return e, nil + if checked { + if e, ok, err := r.reuseLocked(old); ok || err != nil { + return e, err + } + } } tmp, err := os.MkdirTemp("", "syspi-dl-*") if err != nil { @@ -585,6 +652,11 @@ func (r *resolver) resolveRemoteArchive(o *origin.Origin, req request) (*lock.En if o.Hosted() && a.Project.Name != o.Name { return nil, fmt.Errorf("%s: usage %q: the purl names %q but the archive's %s names %q", req.requester, req.resource, o.Name, project.ProjectFile, a.Project.Name) } + known := r.known(id) + known["sha256"] = a.SHA256 + for _, c := range o.Checksums { + known[c.Algorithm] = c.Value + } v, _ := version.Parse(a.Project.Version) if !req.constraint.Check(v) { return nil, fmt.Errorf("%s: usage %q wants %s but the archive holds %s %s", req.requester, req.resource, req.constraint, a.Project.Name, a.Project.Version) diff --git a/internal/install/install_test.go b/internal/install/install_test.go index 3462ce0..76aaadc 100644 --- a/internal/install/install_test.go +++ b/internal/install/install_test.go @@ -640,9 +640,25 @@ func TestHostedArchive(t *testing.T) { t.Fatalf("offline with two checksums %+v", res3) } rewrite(t, dir, project.Usage{Resource: wrong}) - runErr(t, dir, Options{Offline: true}, "the lock records "+sum) + runErr(t, dir, Options{Offline: true}, "checksum mismatch: the resource declares sha256 "+strings.Repeat("00", 32)+", the archive has "+sum) rewrite(t, dir, project.Usage{Resource: id + "&checksum=sha1:" + strings.Repeat("00", 20)}) runErr(t, dir, Options{Offline: true}, "checksum mismatch: the resource declares sha1 "+strings.Repeat("00", 20)) + // A second usage of the same archive is held to its own checksums, even + // when the first already resolved it. + other := t.TempDir() + write(t, other, ".project.json", `{"name":"dev","version":"0.0.1","usage":[{"resource":"`+wrong+`"}]}`) + runErr(t, consumer(t, project.Usage{Resource: good}, project.Usage{Resource: other}), Options{}, "checksum mismatch: the resource declares sha256 "+strings.Repeat("00", 32)+", the archive has "+sum) + write(t, other, ".project.json", `{"name":"dev","version":"0.0.1","usage":[{"resource":"`+id+"&checksum=sha1:"+strings.Repeat("00", 20)+`"}]}`) + runErr(t, consumer(t, project.Usage{Resource: good}, project.Usage{Resource: other}), Options{}, "checksum mismatch: the resource declares sha1 "+strings.Repeat("00", 20)+", the archive has "+sha1sum) + // A lock-only run on a fresh clone cannot check a sha1 against the lock, + // so it downloads the archive to check it rather than trusting the lock. + clone := consumer(t, project.Usage{Resource: id + "&checksum=sha1:" + strings.Repeat("00", 20)}) + write(t, clone, project.LockFile, string(readFile(t, filepath.Join(dir, project.LockFile)))) + runErr(t, clone, Options{LockOnly: true}, "checksum mismatch: the resource declares sha1 "+strings.Repeat("00", 20)+", the archive has "+sha1sum) + rewrite(t, clone, project.Usage{Resource: id + "&checksum=sha1:" + sha1sum}) + if res, _ := run(t, clone, Options{LockOnly: true}); res.Lock.Find(id) == nil || res.Lock.Find(id).SHA256 != sum { + t.Fatalf("lock-only with a sha1 %+v", res.Lock.Find(id)) + } // Without a checksum the download is trusted as a plain IRI is; the lock still pins its digest. plain := consumer(t, project.Usage{Resource: id}) @@ -658,6 +674,16 @@ func TestHostedArchive(t *testing.T) { runErr(t, consumer(t, project.Usage{Resource: good}), Options{Offline: true}, "offline: "+srv.URL+"/release.kpar is not installed") } +// readFile reads a file or fails the test. +func readFile(t *testing.T, p string) []byte { + t.Helper() + data, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return data +} + // rewrite replaces the consumer's usages. func rewrite(t *testing.T, dir string, usages ...project.Usage) { t.Helper() From cac1ed6c8a0e317b4a9e1707952a843cd38be95e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 06:31:24 +0000 Subject: [PATCH 4/4] fix(install): a damaged installed archive is replaced, not held to the declared checksums verifyDeclared reads a copy only when its SHA-256 is the lock's; otherwise the download replaces it and is checked. Co-Authored-By: jason.han --- internal/install/install.go | 6 ++++-- internal/install/install_test.go | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/install/install.go b/internal/install/install.go index 1af94b0..f23be11 100644 --- a/internal/install/install.go +++ b/internal/install/install.go @@ -494,8 +494,10 @@ func (r *resolver) verifyDeclared(o *origin.Origin, e *lock.Entry) (bool, error) if file == "" { return false, nil } - got, err := digest.Sums(file, unchecked) - if err != nil { + // A copy that is not the locked archive (damaged, say) is no witness; the + // download replaces it and is checked instead. + got, err := digest.Sums(file, append(unchecked, digest.Checksum{Algorithm: "sha256", Value: e.SHA256})) + if err != nil || got["sha256"] != e.SHA256 { return false, err } for algo, v := range got { diff --git a/internal/install/install_test.go b/internal/install/install_test.go index 76aaadc..2f30310 100644 --- a/internal/install/install_test.go +++ b/internal/install/install_test.go @@ -643,6 +643,17 @@ func TestHostedArchive(t *testing.T) { runErr(t, dir, Options{Offline: true}, "checksum mismatch: the resource declares sha256 "+strings.Repeat("00", 32)+", the archive has "+sum) rewrite(t, dir, project.Usage{Resource: id + "&checksum=sha1:" + strings.Repeat("00", 20)}) runErr(t, dir, Options{Offline: true}, "checksum mismatch: the resource declares sha1 "+strings.Repeat("00", 20)) + // A damaged installed copy is replaced from the server, not reported + // against the declared sha1. + rewrite(t, dir, project.Usage{Resource: id + "&checksum=sha1:" + sha1sum}) + installed := filepath.Join(dir, "lib", "acme-tools_3.1.0.kpar") + write(t, dir, "lib/acme-tools_3.1.0.kpar", "damaged") + if res, _ := run(t, dir, Options{}); len(res.Installed) != 1 { + t.Fatalf("damaged copy %+v", res) + } + if d, _ := digest.File(installed); d != sum { + t.Fatal("damaged copy not replaced") + } // A second usage of the same archive is held to its own checksums, even // when the first already resolved it. other := t.TempDir()