From cb721a7b8c825685fe37a5c84e6cf0012dd4d0d7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:50:09 +0000 Subject: [PATCH] fix(deps): update module github.com/cbergoon/merkletree to v0.5.0 --- go.mod | 2 +- go.sum | 2 + .../cbergoon/merkletree/.travis.yml | 1 - .../github.com/cbergoon/merkletree/README.md | 311 +++- vendor/github.com/cbergoon/merkletree/doc.go | 118 +- .../cbergoon/merkletree/merkle_tree.go | 1376 +++++++++++++++-- .../github.com/cbergoon/merkletree/proof.go | 230 +++ .../cbergoon/merkletree/serialize.go | 913 +++++++++++ vendor/modules.txt | 4 +- 9 files changed, 2792 insertions(+), 165 deletions(-) delete mode 100644 vendor/github.com/cbergoon/merkletree/.travis.yml create mode 100644 vendor/github.com/cbergoon/merkletree/proof.go create mode 100644 vendor/github.com/cbergoon/merkletree/serialize.go diff --git a/go.mod b/go.mod index 278d1cb2..5fbfef44 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.12 github.com/aws/aws-sdk-go-v2/service/s3 v1.107.1 github.com/aws/smithy-go v1.27.7 - github.com/cbergoon/merkletree v0.2.0 + github.com/cbergoon/merkletree v0.5.0 github.com/cilium/ebpf v0.22.0 github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 github.com/containernetworking/cni v1.3.0 diff --git a/go.sum b/go.sum index d11c0b44..b76af1fe 100644 --- a/go.sum +++ b/go.sum @@ -146,6 +146,8 @@ github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/cbergoon/merkletree v0.2.0 h1:Bttqr3OuoiZEo4ed1L7fTasHka9II+BF9fhBfbNEEoQ= github.com/cbergoon/merkletree v0.2.0/go.mod h1:5c15eckUgiucMGDOCanvalj/yJnD+KAZj1qyJtRW5aM= +github.com/cbergoon/merkletree v0.5.0 h1:570ckfhpRbI6SK1vZddMymVXtKQBnaSKyU49icyIExQ= +github.com/cbergoon/merkletree v0.5.0/go.mod h1:O9XkOuoRsCjlxpEqOEyXeFNe8YxGCnRwYRDt+1waRsc= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= diff --git a/vendor/github.com/cbergoon/merkletree/.travis.yml b/vendor/github.com/cbergoon/merkletree/.travis.yml deleted file mode 100644 index 4f2ee4d9..00000000 --- a/vendor/github.com/cbergoon/merkletree/.travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: go diff --git a/vendor/github.com/cbergoon/merkletree/README.md b/vendor/github.com/cbergoon/merkletree/README.md index 14ab1087..0c47d0fd 100644 --- a/vendor/github.com/cbergoon/merkletree/README.md +++ b/vendor/github.com/cbergoon/merkletree/README.md @@ -1,9 +1,8 @@

Merkle Tree in Golang

-Build -Report -Docs -Version +Build +Docs +Version

An implementation of a Merkle Tree written in Go. A Merkle Tree is a hash tree that provides an efficient way to verify @@ -18,7 +17,248 @@ nlog2(n) steps in the worst case. #### Documentation -See the docs [here](https://godoc.org/github.com/cbergoon/merkletree). +See the docs [here](https://pkg.go.dev/github.com/cbergoon/merkletree). + +#### Constructions + +The tree can be built three ways. All of them are supported; they produce different roots +and are not interchangeable, so pick one when the tree is created. + +| Construction | Built with | Notes | +| --- | --- | --- | +| Bitcoin-style (default) | `NewTree`, `NewTreeWithHashStrategy` | Pairs siblings in order, duplicates the last node on an odd count | +| Sorted siblings | `NewTreeWithHashStrategySorted`, `WithSortedSiblings()` | Orders each pair before hashing, matching OpenZeppelin `MerkleProof` | +| RFC 6962 | `WithRFC6962()` | Prefixed leaf and interior hashes, splits instead of padding | + +The default follows Bitcoin deliberately, so the roots line up with Bitcoin-style trees. +That construction carries two well-known properties: + +**Duplicated odd nodes.** A level holding an odd number of nodes duplicates its last node +so it can be paired, which means a tree built from an odd number of leaves has the same +root as a tree that spells the duplicate out — this is CVE-2012-2459: + +``` +[A, B, C] and [A, B, C, C] -> same root +[A] and [A, A] -> same root +``` + +**No separation between leaf and interior hashes.** Both are computed the same way, so an +interior digest can be handed back as a leaf. A two-leaf tree whose leaves are the two +subtree hashes of a four-leaf tree reproduces the original root exactly, and the forged +tree verifies against itself. + +Sorted mode shares both, and additionally discards leaf order: `[A B C D]`, `[B A C D]` +and `[D C B A]` all produce the same root, because each pair is ordered before it is +hashed. Only regrouping which leaves are paired changes the root. Check +`tree.Sorted()` if you depend on the root committing to order. + +##### RFC 6962 + +`WithRFC6962()` builds the tree specified by +[RFC 6962 section 2.1](https://datatracker.ietf.org/doc/html/rfc6962#section-2.1), which +closes both of the above: + +```go +tree, err := merkletree.NewTreeWithOptions(list, merkletree.WithRFC6962()) +``` + +Leaf hashes are computed as `H(0x00 ‖ digest)` and interior hashes as `H(0x01 ‖ left ‖ right)`, +so a forged leaf would need a genuine collision between two differently prefixed inputs +rather than a rearrangement. Odd node counts are split at the largest power of two below +their length instead of being padded, so every distinct leaf sequence gets a distinct +root and `[A, B, C]` no longer collides with `[A, B, C, C]`. + +Two things to know. RFC 6962 hashes raw leaf data, whereas this tree hashes whatever +`CalculateHash` returns — roots match a Certificate Transparency log only if your +`CalculateHash` returns the leaf bytes themselves rather than a digest of them. The +structural guarantees hold either way. And a single-leaf tree is its own root, so its +audit path is legitimately empty. + +`WithRFC6962()` cannot be combined with `WithSortedSiblings()`; RFC 6962 specifies its own +sibling ordering, and asking for both returns an error. + +#### Parallel construction + +`WithParallelism(n)` builds the tree across up to `n` goroutines, or GOMAXPROCS of them +when `n < 1`. It is off by default. + +```go +tree, err := merkletree.NewTreeWithOptions(list, merkletree.WithParallelism(0)) +``` + +**`Content.CalculateHash` is called concurrently when this is set**, and your +implementation must be safe for that. Nothing else in the package calls it concurrently +and the tree cannot check whether yours is safe, so this requirement is the entire cost +of the option and the reason it is not the default. + +The root is unaffected — every hash lands in a slot fixed by its position, so a tree +built in parallel is byte for byte the tree built serially. + +Whether it pays depends almost entirely on what `CalculateHash` costs. Measured on an +M5 Max, best of 3: + +| Content | Leaves | Serial | Parallel | | +| --- | --- | --- | --- | --- | +| short string | 256 | 26.5µs | 43.7µs | **0.61×** — slower | +| short string | 4,096 | 476µs | 333µs | 1.43× | +| short string | 65,536 | 6.40ms | 2.14ms | 2.99× | +| 4KB blob | 4,096 | 5.21ms | 723µs | **7.21×** | + +Content hashing is always spread across the goroutine budget, since its cost belongs to +you and cannot be guessed here — which is why the option wins big on expensive leaves at +any size, and can lose on a small tree of cheap ones. Interior hashing is our own work +and known to be small, so it stays serial until the tree is large enough to be worth +splitting. Measure before adopting it for small trees. + +The option covers every construction, including `WithRFC6962`: the unbalanced interior +an RFC tree builds forks across goroutines the same way, so an RFC build of 65,536 short +leaves drops from 9.98ms serial to 2.01ms in parallel. + +Parallelism is not recorded when a tree is serialized, since it has no bearing on the +root; a tree read back rebuilds serially unless built again with the option. + +#### Serving proofs + +`GetMerklePath` locates content by scanning every leaf; build with `WithLeafIndex` to +make that one hash and a map probe, or address leaves by position with +`GetMerklePathByIndex`. When proofs are served at rate, the two slices each proof +returns become the bottleneck — the garbage collector turns into the shared resource +every goroutine queues on. The `Append` forms generate the same proofs into buffers you +own, so a server that reuses them allocates nothing per proof: + +```go +var path [][]byte +var index []int64 +for i := range list { + path, index, err = t.AppendMerklePathByIndex(path[:0], index[:0], i) + // path holds the sibling hashes, index which side each sits on +} +``` + +The appended hashes are the tree's own, not copies: treat them as read-only, and copy +any proof that must outlive the next reuse of its buffer. + +The difference is not subtle. At 65,536 leaves a proof appends in ~17ns against ~76ns +returning fresh slices, and under 18 goroutines serving from one shared tree the append +form runs at 2.9ns per proof — 28× the slice-returning form, because with no allocation +there is nothing shared left to queue on. The comparison tables below carry the +cross-library context. + +#### Serialization + +A tree can be written out and read back. What gets written is not the node graph but the +content the tree is rebuilt from: the ordered leaf content, the name of the hash strategy, +the sibling sort flag, and the Merkle root. Everything else is derived, and the recorded +root makes decoding self-checking — a payload that has been altered, or that is decoded +with the wrong hash strategy, fails rather than producing a tree that quietly verifies +against nothing. + +Register your content type and the standard codecs work directly: + +```go +merkletree.RegisterContent(TestContent{}) // needs MarshalBinary/UnmarshalBinary + +err := gob.NewEncoder(&buf).Encode(tree) + +var decoded merkletree.MerkleTree +err = gob.NewDecoder(&buf).Decode(&decoded) +``` + +`MarshalBinary`, `UnmarshalBinary`, `MarshalJSON`, and `UnmarshalJSON` are all available, +so anything built on `encoding.BinaryMarshaler` or `json.Marshaler` works too. + +To avoid the package-level registry entirely — in a library, or for content that already +has an encoding of its own — supply the content codec directly: + +```go +data, err := tree.MarshalWith(func(c merkletree.Content) ([]byte, error) { + return []byte(c.(TestContent).x), nil +}) + +decoded, err := merkletree.UnmarshalWith(data, func(b []byte) (merkletree.Content, error) { + return TestContent{x: string(b)}, nil +}) +``` + +Hash strategies are recorded by name, since a function cannot be serialized. Everything in +the standard library is registered for you; anything else is one call: + +```go +merkletree.RegisterHashStrategy("keccak256", sha3.NewLegacyKeccak256) +tree, err := merkletree.NewTreeWithHashStrategy(list, sha3.NewLegacyKeccak256) +``` + +Note that a tree with reference cycles cannot be handed to a reflection-based codec +directly — `Node` points back at its `Tree` and its `Parent`. Before these marshalers +existed, `gob.Encode(tree)` did not return an error, it crashed the process with a stack +overflow. The marshalers above are the supported path. + +#### Comparison + +Measured against the other Merkle tree libraries in the Go ecosystem — `txaty/go-merkletree`, +`wealdtech/go-merkletree`, `onrik/gomerkle`, `xsleonard/go-merkle` and `jvsteiner/merkle` — +with SHA-256 and the same leaf data throughout. The benchmarks, the methodology and the +caveats are in [`benchmarks/ANALYSIS.md`](benchmarks/ANALYSIS.md); the numbers below are +from an Apple M5 Max on Go 1.26, so treat the ratios as the durable part. + +`txaty/go-merkletree` gets its own column because it is the closest competitor — the only +other library here with parallel construction, and the one whose published comparisons +prompted this exercise. Where it is also the best of the field, the two columns agree. + +| | merkletree | txaty | best of the rest | rank | +|---|---|---|---|---| +| Construction, serial (65,536 leaves) | 7.67 ms | 9.67 ms | onrik 7.61 ms | 2 / 6 | +| Construction, parallel (65,536 leaves) | **3.08 ms** | 6.47 ms | txaty 6.47 ms | **1 / 2** | +| Parallel speedup, 16 KiB leaves | **12.8×** | 11.9× | txaty 11.9× | **1 / 2** | +| Allocations building 65,536 leaves | 131,140 | 328,230 | onrik 131,090 | 2 / 6 | +| Memory building 65,536 leaves | 18.4 MB | 24.8 MB | wealdtech 11.5 MB | 4 / 6 | +| Single proof (65,536 leaves) | **16.7 ns** | 128 ns | txaty 128 ns | **1 / 6** | +| Full proof set (4,096 leaves) | **25.4 ns/proof** | 158 ns/proof | txaty 158 ns/proof | **1 / 6** | +| Verify a proof (65,536 leaves) | **934 ns** | 1,212 ns | onrik 1,108 ns | **1 / 4** | +| Concurrent proofs, 18 goroutines | **2.9 ns/op** | 165 ns/op | jvsteiner 153 ns/op | **1 / 6** | +| Concurrent scaling, 1 → 18 goroutines | **16.4×** | 1.14× | wealdtech 13.1× | **1 / 6** | +| Proof size on the wire, depth 16 | 640 B | 516 B | txaty 516 B | 2 / 2 | + +The proof rows are `AppendMerklePathByIndex` serving into reused buffers, which +allocates nothing per proof; the slice-returning `GetMerklePathByIndex` runs at 76 ns +and the `WithLeafIndex` lookup at 135 ns, still ahead of or level with everything else. +Without any of the three, locating content is a scan and a full proof set is quadratic. +Serial construction is a near-tie because every implementation is waiting on SHA-256 — +at 16 KiB leaves all six land within 3% of each other, and parallelism is the only thing +that still separates them. + +| | merkletree | txaty | wealdtech | onrik | xsleonard | jvsteiner | +|---|---|---|---|---|---|---| +| Proof generation | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | +| Proof by leaf position 1 | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | +| Allocation-free proofs 2 | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Verify without the tree 3 | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | +| Parallel construction | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | +| Pluggable hash | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | +| Sorted siblings (OpenZeppelin) | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | +| RFC 6962 | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Serialization | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Runtime dependencies | none | x/sync | x/crypto | none | none | protobuf | +| Odd leaf count | duplicate 4 | duplicate | pad to 2ⁿ | promote | promote 5 | promote | + +1 Addressing a leaf by position rather than by value, which avoids hashing the +query. txaty offers this only through the precomputed `Proofs` slice in `ModeProofGen`. +It matters more than it looks: at 16 KiB leaves, by position is 95× faster than by value. +2 `AppendMerklePath` and `AppendMerklePathByIndex` write into slices the +caller supplies and keeps, so a proof server reusing its buffers generates proofs without +allocating at all. No other library here exposes the proof buffers; every one of them +returns freshly allocated structures. It is what makes the concurrency row possible — a +proof that allocates nothing shares nothing, so it scales with cores instead of +contending on the garbage collector. +3 A package-level verify taking a proof, a root and the data. onrik's is a +method, so it needs a `Tree` value it does not otherwise use. +4 Or split, under `WithRFC6962`. 5 Or duplicate, with `DoubleOddNodes`. + +That last row decides interoperability, not speed. These libraries agree on the root for +any power-of-two leaf count and split into three groups otherwise, so a proof from one +group will not verify against a root from another. merkletree's default agrees with txaty +and with xsleonard's `DoubleOddNodes`; its `WithRFC6962` mode is checked against the +Certificate Transparency reference vectors in [`oracle/`](oracle/). #### Install ``` @@ -32,6 +272,7 @@ package main import ( "crypto/sha256" + "errors" "log" "github.com/cbergoon/merkletree" @@ -54,7 +295,11 @@ func (t TestContent) CalculateHash() ([]byte, error) { //Equals tests for equality of two Contents func (t TestContent) Equals(other merkletree.Content) (bool, error) { - return t.x == other.(TestContent).x, nil + otherTC, ok := other.(TestContent) + if !ok { + return false, errors.New("value is not of type TestContent") + } + return t.x == otherTC.x, nil } func main() { @@ -99,5 +344,59 @@ func main() { ![merkletree](merkle_tree.png) +#### Testing + +```bash +go test -race ./... # unit, property, golden, and regression tests +scripts/fuzz.sh # every fuzz target, 30s each +scripts/bench.sh # benchmarks, 6 runs each +``` + +The suite checks roots against a reference implementation written directly from each +construction's definition, and pins golden roots and golden serialized payloads that +were produced outside this repository. A change to the hashing or the wire format has +to be made in several independent places before it can pass unnoticed; regenerating a +golden payload to make a test pass is a compatibility break, not a fix. + +##### Fuzzing + +A normal test run only replays each target's seed corpus. `scripts/fuzz.sh` does the +actual fuzzing, one target at a time, discovering targets from the source so a newly +added `FuzzXxx` is picked up automatically: + +```bash +scripts/fuzz.sh # every target, 30s each +scripts/fuzz.sh FuzzUnmarshalBinary # just one +FUZZTIME=5m scripts/fuzz.sh # longer budget per target +``` + +| Target | What it drives | +| --- | --- | +| `FuzzTreeInvariants` | Tree construction, verification, and proof replay across every construction | +| `FuzzUnmarshalBinary` | The binary decoder: no faults, decoded trees verify, payloads are canonical | +| `FuzzUnmarshalJSON` | The JSON decoder | +| `FuzzPayloadCorruption` | That an altered payload can never decode to a different Merkle root | + +CI fuzzes on every pull request for 60s per target, and on every merge to `master` for +180s per target. Both share a corpus cached between runs, so the search accumulates +across builds rather than restarting each time. A failing input is written to +`testdata/fuzz//` and uploaded as a build artifact; committing that file turns +the finding into a permanent regression test. + +##### Benchmarks + +`scripts/bench.sh` covers construction, verification, proof generation, rebuilds, and +all three codecs, across each construction and a range of leaf counts. Timings are +noisy, so it runs each benchmark six times by default and can hand the results to +benchstat: + +```bash +scripts/bench.sh # everything +scripts/bench.sh Verify # only benchmarks matching /Verify/ +scripts/bench.sh -o before.txt # save a baseline +scripts/bench.sh -o after.txt -b before.txt # ...and compare against it +scripts/bench.sh -s # smoke: one iteration, just checks they run +``` + #### License This project is licensed under the MIT License. diff --git a/vendor/github.com/cbergoon/merkletree/doc.go b/vendor/github.com/cbergoon/merkletree/doc.go index 5da0be47..e4338a89 100644 --- a/vendor/github.com/cbergoon/merkletree/doc.go +++ b/vendor/github.com/cbergoon/merkletree/doc.go @@ -1,7 +1,8 @@ // Copyright 2017 Cameron Bergoon // Licensed under the MIT License, see LICENCE file for details. -/*Package merkletree implements a Merkle Tree capable of storing arbitrary content. +/* +Package merkletree implements a Merkle Tree capable of storing arbitrary content. A Merkle Tree is a hash tree that provides an efficient way to verify the contents of a set data are present and untampered with. At its core, a Merkle Tree is @@ -17,14 +18,121 @@ Creating a new merkletree requires that the type that the tree will be construct from implements the Content interface. type Content interface { - CalculateHash() []byte - Equals(other Content) bool + CalculateHash() ([]byte, error) + Equals(other Content) (bool, error) } A slice of the Content items should be created and then passed to the NewTree method. - t, err := merkle.NewTree(list) + t, err := merkletree.NewTree(list) t represents the Merkle Tree and can be verified and manipulated with the API methods -described below.*/ +described below. + +# Constructions + +By default the tree is built the way Bitcoin builds one: sibling hashes are concatenated +in the order given, and a level holding an odd number of nodes duplicates its last node +so it can be paired. NewTreeWithHashStrategySorted additionally orders each pair of +siblings before hashing, matching the OpenZeppelin MerkleProof convention. + +Both of those constructions inherit two well known properties. The last node duplication +means [A B C] and [A B C C] hash to the same root, which is CVE-2012-2459. And because +leaf and interior hashes are computed the same way, an interior digest can be presented +as a leaf: a two leaf tree whose leaves are the two subtree hashes of a four leaf tree +reproduces the original root, and verifies. + +NewTreeWithOptions with WithRFC6962 builds the tree specified by RFC 6962 section 2.1 +instead, which closes both. Leaf and interior hashes carry distinct one byte prefixes, +and node lists are split at the largest power of two below their length rather than +padded, so every distinct leaf sequence has a distinct root. + + t, err := merkletree.NewTreeWithOptions(list, merkletree.WithRFC6962()) + +The constructions produce different roots and are not interchangeable, so pick one when +the tree is created. See WithRFC6962 for the details, including how it relates to +Certificate Transparency. + +# Parallel construction + +WithParallelism builds a tree across several goroutines. It is off by default, because +it calls Content.CalculateHash concurrently and only the caller knows whether their +implementation is safe for that. + + t, err := merkletree.NewTreeWithOptions(list, merkletree.WithParallelism(0)) + +The root is unaffected: results are written to slots fixed by position, so a tree built +in parallel is byte for byte the tree built serially. What it is worth depends on what +CalculateHash costs, and is large when content is expensive to hash and negative on a +small tree of cheap content. See WithParallelism. + +# Proof lookup + +GetMerklePath and VerifyContent locate their content by scanning every leaf, which makes +a single proof cost O(n) in the leaf count even though the walk it performs afterwards is +only O(log n), and a full set of proofs O(n²). Two options avoid the scan. + +WithLeafIndex records a lookup table from leaf hash to leaf position at construction, so +a lookup becomes one hash and one map probe whatever the leaf count: + + t, err := merkletree.NewTreeWithOptions(list, merkletree.WithLeafIndex()) + +It costs memory proportional to the leaf count and changes how content is located, from +Content.Equals to a comparison of hashes. The Content interface already requires the two +to agree, so any implementation honoring that contract gets the same leaf either way. See +WithLeafIndex. + +GetMerklePathByIndex takes the position in Leafs directly, needs no option and no extra +memory, and is the better choice when the caller already tracks which item is which. + +AppendMerklePath and AppendMerklePathByIndex produce the same proofs into caller +supplied slices, so a proof server reusing its buffers generates proofs without +allocating at all: + + path, index, err = t.AppendMerklePathByIndex(path[:0], index[:0], i) + +# Verifying a proof without the tree + +VerifyProof checks an audit path against a root and needs no tree, which is the operation +a verifier performs when it holds a root from a source it trusts and a proof from one it +does not: + + ok, err := merkletree.VerifyProof(content, path, index, root, merkletree.WithRFC6962()) + +The options must describe the construction the proof was produced under. A verifier that +has no Content implementation can use VerifyProofWithDigest instead, and one that does +hold the tree can call the MerkleTree.VerifyProof method and skip restating the options. + +This is a different question from MerkleTree.VerifyContent, which walks a tree it already +has and recomputes every hash on the path from its own stored nodes. VerifyContent is the +stronger check where the tree is available; VerifyProof is the only one available where +it is not. See VerifyProof for what a verified proof does and does not establish, and why +WithRFC6962 matters more for proofs from untrusted sources. + +# Serialization + +A tree holds reference cycles - a Node points back at its Tree and at its Parent - so it +cannot be handed to a reflection-based codec directly. Use the marshalers instead, which +write the content the tree is rebuilt from rather than the node graph: the ordered leaf +content, the name of the hash strategy, the construction the tree was built with, and the +Merkle root. The recorded root is checked against the rebuilt tree on decode, so a payload +that has been tampered with, truncated, or decoded with the wrong hash strategy is +rejected. + +Register the content type and encoding/gob, encoding/json, and anything else built on +encoding.BinaryMarshaler or json.Marshaler will work: + + merkletree.RegisterContent(MyContent{}) + err := gob.NewEncoder(&buf).Encode(t) + +Content types registered this way must implement encoding.BinaryMarshaler, and a pointer +to them must implement encoding.BinaryUnmarshaler. + +To serialize without touching a package-level registry, supply the content codec directly +with MarshalWith and UnmarshalWith. + +Hash strategies are recorded by name, because a function value cannot be serialized. The +standard library strategies are registered automatically; register anything else, such as +keccak256 or blake2b, with RegisterHashStrategy before marshaling a tree that uses it. +*/ package merkletree diff --git a/vendor/github.com/cbergoon/merkletree/merkle_tree.go b/vendor/github.com/cbergoon/merkletree/merkle_tree.go index ae2e2769..435def49 100644 --- a/vendor/github.com/cbergoon/merkletree/merkle_tree.go +++ b/vendor/github.com/cbergoon/merkletree/merkle_tree.go @@ -9,26 +9,167 @@ import ( "errors" "fmt" "hash" + "math/bits" + "runtime" + "strings" + "sync" ) -//Content represents the data that is stored and verified by the tree. A type that -//implements this interface can be used as an item in the tree. +var ( + // ErrNoContent is returned when a tree is built from an empty list of content. + ErrNoContent = errors.New("error: cannot construct tree with no content") + + // ErrNilContent is returned when a list of content holds a nil entry. + ErrNilContent = errors.New("error: cannot construct tree with nil content") + + // ErrContentNotFound is returned by GetMerklePath when the requested content is + // not held by any leaf of the tree. Test for it with errors.Is. + ErrContentNotFound = errors.New("error: content not found in tree") + + // ErrMalformedTree is returned by the verification methods when the tree they + // are asked to walk is not one a constructor could have produced: a zero value + // MerkleTree, or a node graph whose interior nodes are missing a child or a tree + // back-pointer. Node exposes its fields, so a caller can assemble such a graph by + // hand or by editing a decoded tree; verification reports it rather than faulting. + ErrMalformedTree = errors.New("error: tree is empty or malformed") +) + +// Content represents the data that is stored and verified by the tree. A type that +// implements this interface can be used as an item in the tree. +// +// Implementations are expected to keep Equals and CalculateHash consistent: two +// items that report equal should hash equal, and two items that hash equal should +// report equal. Lookups by content return the first matching leaf, so a type that +// breaks this correspondence can be located by one method and hashed by the other. type Content interface { CalculateHash() ([]byte, error) Equals(other Content) (bool, error) } -//MerkleTree is the container for the tree. It holds a pointer to the root of the tree, -//a list of pointers to the leaf nodes, and the merkle root. +// MerkleTree is the container for the tree. It holds a pointer to the root of the tree, +// a list of pointers to the leaf nodes, and the merkle root. +// +// Note that Node points back at its Tree and at its Parent, so the tree contains +// reference cycles and cannot be handed directly to a reflection-based codec. Use the +// marshalers in serialize.go, which encode the content the tree is rebuilt from rather +// than the node graph itself. type MerkleTree struct { - Root *Node - merkleRoot []byte - Leafs []*Node - hashStrategy func() hash.Hash + Root *Node + merkleRoot []byte + Leafs []*Node + // hashStrategyName is the name hashStrategy is registered under, when it is + // known. It is only ever set by the unmarshalers, to preserve the name a payload + // was written with even when the strategy was supplied via WithHashStrategy + // rather than resolved through the registry. When it is empty the name is looked + // up from hashStrategy at marshal time, so registering a strategy after building + // a tree with it still works. + hashStrategyName string + hashStrategy func() hash.Hash + sort bool + rfc6962 bool + // parallelism is the goroutine budget for building this tree, or zero to build + // serially. Unlike sort and rfc6962 it does not affect the root, so it is a + // property of how a tree is built rather than of the tree itself, and it is + // deliberately absent from the serialized form. + parallelism int + // wantLeafIndex records that WithLeafIndex was asked for, so that a rebuild + // regenerates the index rather than silently dropping it. Like parallelism it + // does not affect the root and is absent from the serialized form. + wantLeafIndex bool + // leafIndex maps a leaf hash to the lowest index in Leafs holding it, or is nil + // when the tree was built without WithLeafIndex. It is written only while a tree + // is being built or rebuilt and is read only afterwards, so proof serving needs + // no synchronization. + leafIndex map[string]int + // scratchPool recycles the hasher and replay buffer one proof verification + // needs, or is nil, in which case they are created per call. Only + // defaultProofConfig carries one: its strategy is fixed forever, so a recycled + // hasher can never be of the wrong kind. Trees deliberately have none, because + // hashStrategy is a mutable field within this package and a pool filled before + // a swap would keep hashing with the old strategy. Held by pointer so that + // copying a MerkleTree, as the value-receiver marshalers do, copies a reference + // rather than the pool's internals. + scratchPool *sync.Pool +} + +// Prefixes that separate the two kinds of hash an RFC 6962 tree computes, so that no +// interior digest can ever be mistaken for a leaf digest. +const ( + rfc6962LeafPrefix = 0x00 + rfc6962InteriorPrefix = 0x01 +) + +// The prefixes as slices, so writing one does not allocate on every node. +var ( + rfc6962LeafPrefixBytes = []byte{rfc6962LeafPrefix} + rfc6962InteriorPrefixBytes = []byte{rfc6962InteriorPrefix} +) + +// hashLeaf produces the hash recorded on the leaf holding c. +// +// In the default construction this is whatever Content.CalculateHash returned. Under +// RFC 6962 that digest is hashed again behind a leaf prefix, which is what stops an +// interior digest being presented as a leaf. +func (m *MerkleTree) hashLeaf(c Content) ([]byte, error) { + digest, err := c.CalculateHash() + if err != nil { + return nil, err + } + if !m.rfc6962 { + return digest, nil + } + + return m.appendLeafDigest(m.hashStrategy(), nil, digest) +} + +// appendLeafDigest appends the RFC 6962 leaf hash of digest to dst, reusing h. +// +// The hasher must not be shared across goroutines. Construction keeps its hasher local +// to the call, and VerifyTree creates one for the walk it then runs to completion on a +// single goroutine, so concurrent reads of a built tree remain safe. +func (m *MerkleTree) appendLeafDigest(h hash.Hash, dst, digest []byte) ([]byte, error) { + h.Reset() + if _, err := h.Write(rfc6962LeafPrefixBytes); err != nil { + return nil, err + } + if _, err := h.Write(digest); err != nil { + return nil, err + } + + return h.Sum(dst), nil +} + +// hashInterior produces the hash recorded on the interior node above left and right. +func (m *MerkleTree) hashInterior(left, right []byte) ([]byte, error) { + return m.appendInteriorHash(m.hashStrategy(), nil, left, right) +} + +// appendInteriorHash appends the interior hash of left and right to dst, reusing h. +// +// The pair is written to the hasher in two calls rather than being concatenated into a +// scratch slice first. The bytes fed to the hash are identical either way, so roots are +// unaffected, but the concatenation is one allocation per interior node. +func (m *MerkleTree) appendInteriorHash(h hash.Hash, dst, left, right []byte) ([]byte, error) { + h.Reset() + if m.rfc6962 { + if _, err := h.Write(rfc6962InteriorPrefixBytes); err != nil { + return nil, err + } + } else { + left, right = sortPair(m.sort, left, right) + } + if _, err := h.Write(left); err != nil { + return nil, err + } + if _, err := h.Write(right); err != nil { + return nil, err + } + + return h.Sum(dst), nil } -//Node represents a node, root, or leaf in the tree. It stores pointers to its immediate -//relationships, a hash, the content stored if it is a leaf, and other metadata. +// Node represents a node, root, or leaf in the tree. It stores pointers to its immediate +// relationships, a hash, the content stored if it is a leaf, and other metadata. type Node struct { Tree *MerkleTree Parent *Node @@ -40,49 +181,181 @@ type Node struct { C Content } -//verifyNode walks down the tree until hitting a leaf, calculating the hash at each level -//and returning the resulting hash of Node n. -func (n *Node) verifyNode() ([]byte, error) { +// sortAppend concatenates a and b, optionally ordering the pair by big-endian +// integer value first so the result matches the OpenZeppelin MerkleProof convention. +// https://github.com/OpenZeppelin/openzeppelin-contracts/blob/master/contracts/utils/cryptography/MerkleProof.sol +// +// The result is always a freshly allocated slice. Appending directly onto a or b +// would write into their spare capacity, and Content.CalculateHash may legally +// return a slice with cap > len that the caller still owns. The concatenated bytes +// are identical either way, so Merkle roots are unaffected. +func sortAppend(sort bool, a, b []byte) []byte { + a, b = sortPair(sort, a, b) + + out := make([]byte, 0, len(a)+len(b)) + out = append(out, a...) + return append(out, b...) +} + +// sortPair returns a and b in the order they should be hashed: unchanged by default, +// or ordered by big-endian integer value when sort is set. It is the ordering half of +// sortAppend, split out so the hashing path can order a pair without concatenating it. +func sortPair(sort bool, a, b []byte) ([]byte, []byte) { + if sort && compareBigEndian(a, b) != -1 { + return b, a + } + + return a, b +} + +// compareBigEndian compares a and b as big-endian unsigned integers, returning -1, 0 +// or 1 the way big.Int.Cmp does. +// +// This reproduces big.Int comparison exactly rather than approximating it with +// bytes.Compare. The two agree only for equal-length inputs: read as integers, +// {0x01, 0x00} is larger than {0xff}, while lexicographically it is smaller. Digests +// from a single hash strategy are all the same width, but Content.CalculateHash is +// free to return any width, and the ordering chosen here decides the Merkle root. +// +// Going through big.Int would cost two heap allocations per interior node, which on a +// sorted tree is the largest single source of allocation during construction. +func compareBigEndian(a, b []byte) int { + // Leading zero bytes do not change the value, so {0x00, 0x01} and {0x01} are + // equal here just as they are to big.Int. + for len(a) > 0 && a[0] == 0 { + a = a[1:] + } + for len(b) > 0 && b[0] == 0 { + b = b[1:] + } + if len(a) != len(b) { + if len(a) < len(b) { + return -1 + } + + return 1 + } + + return bytes.Compare(a, b) +} + +// verifyNode walks down the tree until hitting a leaf, recalculating the hash at each +// level from the content beneath it. It appends the recalculated hash of Node n to dst +// and returns the extended buffer along with whether n and everything below it matched +// the hash each node has recorded. The recalculated hash is the tail of the returned +// buffer, from the length dst had on entry. +// +// Recalculating alone only proves the content still hashes to the root; it says nothing +// about the hashes cached on the nodes in between. Reporting both lets VerifyTree catch +// a tree whose interior hashes have been edited as well as one whose content has. +// +// The hasher and the buffer are threaded through the whole walk rather than being +// created per node. A fresh hasher and a fresh digest for every node made allocation, +// not hashing, the dominant cost of verifying: the node count is linear in the leaves, +// so a tree of any size paid two allocations per node. Both are owned by the caller for +// the duration of the walk, which is single goroutine, so nothing here is shared. +func (n *Node) verifyNode(h hash.Hash, dst []byte) ([]byte, bool, error) { + // Hashing anything at all needs the tree's strategy, and an interior node needs + // both children. Neither can be missing in a tree a constructor built, but the + // fields are exported, so check rather than fault on a graph assembled by hand. + if n.Tree == nil { + return nil, false, fmt.Errorf("%w: node is not attached to a tree", ErrMalformedTree) + } if n.leaf { - return n.C.CalculateHash() + off := len(dst) + digest, err := n.C.CalculateHash() + if err != nil { + return nil, false, err + } + // Only RFC 6962 hashes the leaf digest again; the default construction + // records what CalculateHash returned. This mirrors hashLeaf, but appends + // into the shared buffer rather than allocating. + if n.Tree.rfc6962 { + if dst, err = n.Tree.appendLeafDigest(h, dst, digest); err != nil { + return nil, false, err + } + } else { + dst = append(dst, digest...) + } + + return dst, bytes.Equal(dst[off:], n.Hash), nil + } + if n.Left == nil || n.Right == nil { + return nil, false, fmt.Errorf("%w: interior node is missing a child", ErrMalformedTree) } - rightBytes, err := n.Right.verifyNode() + + off := len(dst) + dst, rightMatched, err := n.Right.verifyNode(h, dst) if err != nil { - return nil, err + return nil, false, err } + mid := len(dst) - leftBytes, err := n.Left.verifyNode() + dst, leftMatched, err := n.Left.verifyNode(h, dst) if err != nil { - return nil, err + return nil, false, err } + end := len(dst) - h := n.Tree.hashStrategy() - if _, err := h.Write(append(leftBytes, rightBytes...)); err != nil { - return nil, err + // The children's digests are read into the hasher before Sum appends, so a growth + // of dst here cannot invalidate them. + dst, err = n.Tree.appendInteriorHash(h, dst, dst[mid:end], dst[off:mid]) + if err != nil { + return nil, false, err } - return h.Sum(nil), nil + // The children's digests are dead once this node's is computed, so slide it down + // over them. The buffer then holds only the path currently being walked, which + // keeps it at the depth of the tree rather than the size of it. + dst = dst[:off+copy(dst[off:], dst[end:])] + + return dst, leftMatched && rightMatched && bytes.Equal(dst[off:], n.Hash), nil } -//calculateNodeHash is a helper function that calculates the hash of the node. -func (n *Node) calculateNodeHash() ([]byte, error) { - if n.leaf { - return n.C.CalculateHash() +// calculateNodeHash is a helper function that calculates the hash of the node. +// +// The argument is retained so existing callers keep compiling. The tree's own sort +// setting is what gets applied, and every call site has always passed exactly that. +func (n *Node) calculateNodeHash(_ bool) ([]byte, error) { + if n.Tree == nil { + return nil, fmt.Errorf("%w: node is not attached to a tree", ErrMalformedTree) } - h := n.Tree.hashStrategy() - if _, err := h.Write(append(n.Left.Hash, n.Right.Hash...)); err != nil { - return nil, err + return n.appendCalculatedHash(n.Tree.hashStrategy(), nil) +} + +// appendCalculatedHash appends the hash calculateNodeHash would return to dst, +// reusing h instead of creating a hasher and a digest per node. It is what lets +// VerifyContent walk a whole path with one hasher and one buffer, the same way +// verifyNode does; a fresh pair per node made allocation the dominant cost of both. +func (n *Node) appendCalculatedHash(h hash.Hash, dst []byte) ([]byte, error) { + if n.Tree == nil { + return nil, fmt.Errorf("%w: node is not attached to a tree", ErrMalformedTree) + } + if n.leaf { + digest, err := n.C.CalculateHash() + if err != nil { + return nil, err + } + if n.Tree.rfc6962 { + return n.Tree.appendLeafDigest(h, dst, digest) + } + + return append(dst, digest...), nil + } + if n.Left == nil || n.Right == nil { + return nil, fmt.Errorf("%w: interior node is missing a child", ErrMalformedTree) } - return h.Sum(nil), nil + return n.Tree.appendInteriorHash(h, dst, n.Left.Hash, n.Right.Hash) } -//NewTree creates a new Merkle Tree using the content cs. +// NewTree creates a new Merkle Tree using the content cs. func NewTree(cs []Content) (*MerkleTree, error) { var defaultHashStrategy = sha256.New t := &MerkleTree{ hashStrategy: defaultHashStrategy, + sort: false, } root, leafs, err := buildWithContent(cs, t) if err != nil { @@ -94,12 +367,13 @@ func NewTree(cs []Content) (*MerkleTree, error) { return t, nil } -//NewTreeWithHashStrategy creates a new Merkle Tree using the content cs using the provided hash -//strategy. Note that the hash type used in the type that implements the Content interface must -//match the hash type profided to the tree. +// NewTreeWithHashStrategy creates a new Merkle Tree using the content cs using the provided hash +// strategy. Note that the hash type used in the type that implements the Content interface must +// match the hash type provided to the tree. func NewTreeWithHashStrategy(cs []Content, hashStrategy func() hash.Hash) (*MerkleTree, error) { t := &MerkleTree{ hashStrategy: hashStrategy, + sort: false, } root, leafs, err := buildWithContent(cs, t) if err != nil { @@ -111,67 +385,606 @@ func NewTreeWithHashStrategy(cs []Content, hashStrategy func() hash.Hash) (*Merk return t, nil } -// GetMerklePath: Get Merkle path and indexes(left leaf or right leaf) -func (m *MerkleTree) GetMerklePath(content Content) ([][]byte, []int64, error) { - for _, current := range m.Leafs { - ok, err := current.C.Equals(content) +// NewTreeWithHashStrategySorted just like NewTreeWithHashStrategy +// but sorts the siblings before hashing, mostly to follow the OpenZepplin Merkle implementation +// https://github.com/OpenZeppelin/openzeppelin-contracts-ethereum-package/blob/master/contracts/cryptography/MerkleProof.sol +func NewTreeWithHashStrategySorted(cs []Content, hashStrategy func() hash.Hash, sort bool) (*MerkleTree, error) { + t := &MerkleTree{ + hashStrategy: hashStrategy, + sort: sort, + } + root, leafs, err := buildWithContent(cs, t) + if err != nil { + return nil, err + } + t.Root = root + t.Leafs = leafs + t.merkleRoot = root.Hash + return t, nil +} + +// TreeOption configures a tree built by NewTreeWithOptions. +type TreeOption func(*MerkleTree) + +// WithHasher sets the hash strategy used for interior nodes, and for the leaf prefix +// under WithRFC6962. It defaults to sha256.New. +// +// The hash a Content implementation returns from CalculateHash must be produced by a +// compatible algorithm; the tree does not and cannot check this. +func WithHasher(strategy func() hash.Hash) TreeOption { + return func(m *MerkleTree) { + m.hashStrategy = strategy + } +} + +// WithSortedSiblings orders each pair of siblings by their big-endian integer value +// before hashing them, matching the OpenZeppelin MerkleProof convention. It is the +// option form of NewTreeWithHashStrategySorted. +// +// Sorting makes the root independent of the order content was supplied in; see +// MerkleTree.Sorted. It cannot be combined with WithRFC6962, which specifies its own +// unsorted ordering. +func WithSortedSiblings() TreeOption { + return func(m *MerkleTree) { + m.sort = true + } +} + +// WithRFC6962 builds the tree the way RFC 6962 section 2.1 specifies, which closes two +// weaknesses in the default Bitcoin-style construction: +// +// Leaf and interior hashes are computed behind distinct one byte prefixes, 0x00 and +// 0x01. Without them an interior digest can be handed back as a leaf: a two leaf tree +// whose leaves are the two subtree hashes of a four leaf tree reproduces the original +// root exactly, and the forged tree verifies. With them a forgery would need a genuine +// collision between two differently prefixed inputs. +// +// Odd node counts are split rather than duplicated. The default construction pairs the +// last node with itself, so [A B C] and [A B C C] hash alike, which is CVE-2012-2459. +// RFC 6962 splits each node list at the largest power of two below its length instead, +// so every distinct leaf sequence has a distinct root. +// +// Note that RFC 6962 hashes raw leaf data, whereas this tree hashes whatever +// Content.CalculateHash returns. Roots therefore match a Certificate Transparency log +// only if CalculateHash returns the leaf bytes themselves rather than a digest of +// them. The structural guarantees above hold either way. +// +// Trees built this way do not share roots with trees built any other way, so this is a +// choice to make when a tree is created rather than one to change later. +func WithRFC6962() TreeOption { + return func(m *MerkleTree) { + m.rfc6962 = true + } +} + +// WithParallelism builds the tree using up to n goroutines, or GOMAXPROCS of them when +// n is less than one. It is off by default. +// +// Content.CalculateHash is called concurrently from several goroutines when this is +// set, and implementations must be safe for that. Nothing else in this package calls +// it concurrently and the tree cannot check whether yours is safe, so this requirement +// is the whole cost of the option and the reason it is not the default. +// +// The root is unaffected. Every hash is written to a slot fixed by its position, so the +// order the goroutines finish in cannot change the result: a tree built with this option +// is byte for byte the tree built without it. +// +// How much it buys depends almost entirely on what CalculateHash costs. Hashing the +// interior of the tree parallelizes too, but that work is small and fixed, so it is left +// serial below a threshold where the coordination would cost more than it saves. Content +// hashing is neither small nor knowable in advance, so it is always spread across the +// goroutine budget when this option is set, whatever the leaf count. +// +// That last point cuts both ways, and it is worth measuring rather than assuming. Where +// content is expensive to hash this option is worth several times the serial build, and +// the advantage grows with the cost. Where content is cheap the leaves finish so quickly +// that handing them out costs more than it saves on a small tree: a few thousand leaves +// of a short string is roughly the break-even, and below it a parallel build can be +// slower than a serial one. +// +// Parallelism is not recorded when a tree is serialized, because it has no bearing on +// the root. A tree read back with UnmarshalBinary or UnmarshalJSON rebuilds serially +// unless it is built again with this option. +func WithParallelism(n int) TreeOption { + return func(m *MerkleTree) { + if n < 1 { + n = runtime.GOMAXPROCS(0) + } + m.parallelism = n + } +} + +// WithLeafIndex builds a lookup table from leaf hash to leaf position, so that +// GetMerklePath and VerifyContent find their content by hashing it once instead of +// scanning every leaf. It is off by default. +// +// Without it, locating content is a linear scan calling Content.Equals on each leaf, +// which makes a single proof O(n) and a full set of proofs O(n²). At a hundred thousand +// leaves that is around 98µs per proof and roughly ten seconds for the whole set. With +// it, a lookup is one hash and one map probe whatever the leaf count. +// +// The index costs memory proportional to the leaf count, on the order of the digest +// width plus map overhead per leaf, which is why it is opt in rather than automatic. It +// costs no extra hashing to build: every leaf hash it records was already computed and +// stored during construction. +// +// It changes how content is located, which is the reason to think before reaching for +// it. Without the index a leaf is found by Content.Equals; with it a leaf is found by +// comparing the hash of the query against the recorded leaf hashes. The Content +// interface already requires the two to agree - items that are equal must hash equal, +// and items that hash equal must be equal - so for any implementation that honors that +// contract both routes return the same leaf, including the rule that the earliest +// matching leaf wins. For an implementation that does not, the two disagree, and only +// the scan reflects Equals. +// +// One consequence is worth stating plainly, because it shows up in error handling: an +// indexed lookup hashes the content it is given and never calls Equals, where a scan +// calls Equals on each leaf and never hashes what it was given. A query whose +// CalculateHash returns an error therefore fails only on the indexed path, and content +// whose Equals returns an error surfaces that only on the scanning path. +// +// The root is unaffected, and neither is anything a proof is checked against. Like +// WithParallelism the index is absent from the serialized form, so a tree read back +// with UnmarshalBinary or UnmarshalJSON has no index unless it is asked for again; +// RebuildTree and RebuildTreeWith do regenerate it. +func WithLeafIndex() TreeOption { + return func(m *MerkleTree) { + m.wantLeafIndex = true + } +} + +// NewTreeWithOptions creates a new Merkle Tree using the content cs, configured by the +// given options. With no options it is equivalent to NewTree. +func NewTreeWithOptions(cs []Content, opts ...TreeOption) (*MerkleTree, error) { + // Resolved through the same function the proof verifier uses, so that a tree and a + // proof checked against it cannot disagree about what an option meant. + t, err := configFromOptions(opts) + if err != nil { + return nil, err + } + + root, leafs, err := buildWithContent(cs, t) + if err != nil { + return nil, err + } + t.Root = root + t.Leafs = leafs + t.merkleRoot = root.Hash + t.buildLeafIndex() + + return t, nil +} + +// buildLeafIndex regenerates leafIndex from the current leaves, or clears it when the +// tree was not asked for one. It is called after every build, so a rebuilt tree neither +// keeps a stale index nor silently loses the one it was asked for. +// +// The padding leaf that an odd content count produces carries the same hash as the leaf +// it copies. Keeping the lowest index for a hash means the copy never displaces the +// original, which is the same "earliest leaf wins" rule the scan follows. +func (m *MerkleTree) buildLeafIndex() { + if !m.wantLeafIndex { + m.leafIndex = nil + + return + } + + // The keys are substrings of one backing string holding every leaf hash, rather + // than a string(l.Hash) conversion per leaf. The conversion copies, so building + // the index that way cost one allocation per leaf - the single largest source of + // allocation in an indexed build. Every key is retained by the map anyway, so + // sharing one backing keeps the same bytes alive in one object instead of n. + total := 0 + for _, l := range m.Leafs { + total += len(l.Hash) + } + var b strings.Builder + b.Grow(total) + for _, l := range m.Leafs { + b.Write(l.Hash) + } + keys := b.String() + + idx := make(map[string]int, len(m.Leafs)) + off := 0 + for i, l := range m.Leafs { + k := keys[off : off+len(l.Hash)] + off += len(l.Hash) + if _, seen := idx[k]; !seen { + idx[k] = i + } + } + m.leafIndex = idx +} + +// findLeaf returns the position in Leafs of the first leaf holding content, or -1 if no +// leaf holds it. It goes through the index when the tree has one and falls back to the +// scan over Content.Equals when it does not. +func (m *MerkleTree) findLeaf(content Content) (int, error) { + if m.leafIndex != nil { + // The recorded leaf hashes are what hashLeaf produces, prefix and all under + // RFC 6962, so the query has to go through the same function to be comparable. + digest, err := m.hashLeaf(content) if err != nil { - return nil, nil, err + return -1, err } + // Indexing a map with string(byteSlice) does not allocate the string. + if i, ok := m.leafIndex[string(digest)]; ok { + return i, nil + } + + return -1, nil + } + for i, l := range m.Leafs { + ok, err := l.C.Equals(content) + if err != nil { + return -1, err + } if ok { - currentParent := current.Parent - var merklePath [][]byte - var index []int64 - for currentParent != nil { - if bytes.Equal(currentParent.Left.Hash, current.Hash) { - merklePath = append(merklePath, currentParent.Right.Hash) - index = append(index, 1) // right leaf - } else { - merklePath = append(merklePath, currentParent.Left.Hash) - index = append(index, 0) // left leaf + return i, nil + } + } + + return -1, nil +} + +// pathFromLeaf walks from a leaf up to the root, collecting the sibling hash at each +// level and which side it sits on. +func (m *MerkleTree) pathFromLeaf(current *Node) ([][]byte, []int64) { + // The walk takes one step per level, so the depth of the tree is how many entries + // the two slices end up holding. bits.Len of the highest leaf position is that + // depth exactly, for a power-of-two count as much as a padded or split one, so + // the slices come out exact-fit; this is the hottest allocation site in the + // package, and a spare entry per proof is measurable at proof-serving rates. + depth := bits.Len(uint(len(m.Leafs) - 1)) + + return m.appendPathFromLeaf(make([][]byte, 0, depth), make([]int64, 0, depth), current) +} + +// appendPathFromLeaf is pathFromLeaf appending into caller supplied slices. It is the +// walk both the slice-returning and the appending proof methods share. +func (m *MerkleTree) appendPathFromLeaf(merklePath [][]byte, index []int64, current *Node) ([][]byte, []int64) { + for currentParent := current.Parent; currentParent != nil; currentParent = current.Parent { + // Whether current sits on the left or the right is a structural question, so + // compare node identity rather than hashes. Comparing hashes gives the same + // answer only while Content.Equals agrees with Content.CalculateHash: an + // implementation where two distinct items hash alike would see a right-hand + // node reported as a left-hand one, producing a correct path with a wrong + // index. + if currentParent.Left == current { + merklePath = append(merklePath, currentParent.Right.Hash) + index = append(index, 1) // right leaf + } else { + merklePath = append(merklePath, currentParent.Left.Hash) + index = append(index, 0) // left leaf + } + current = currentParent + } + + return merklePath, index +} + +// GetMerklePath returns the sibling hashes on the path from the leaf holding content +// up to the root, together with an index describing which side each sibling sits on: +// 1 when the sibling is the right hand node, 0 when it is the left hand node. +// +// Content is located by Content.Equals and the first matching leaf wins, so a value +// stored in more than one leaf yields the proof for the earliest of them. If no leaf +// holds the content, the returned error wraps ErrContentNotFound. +// +// Locating the content is a scan over every leaf, which makes this O(n) in the leaf +// count while the walk it performs afterwards is only O(log n). Build the tree with +// WithLeafIndex to replace the scan with a single hash and map probe, or use +// GetMerklePathByIndex when the position is already known. AppendMerklePath produces +// the same proof into caller supplied slices, for callers generating proofs at a rate +// where the two returned slices are a cost worth removing. +func (m *MerkleTree) GetMerklePath(content Content) ([][]byte, []int64, error) { + i, err := m.findLeaf(content) + if err != nil { + return nil, nil, err + } + if i < 0 { + return nil, nil, ErrContentNotFound + } + + merklePath, index := m.pathFromLeaf(m.Leafs[i]) + + return merklePath, index, nil +} + +// GetMerklePathByIndex returns the same audit path as GetMerklePath for the leaf at +// position i in Leafs, without locating it by content first. It costs one step per +// level of the tree whatever the leaf count, where GetMerklePath has to find the leaf +// before it can walk. +// +// The index is a position in Leafs, which is the order the content was supplied in. +// Note that a tree built from an odd number of items holds one more leaf than it was +// given, the last being a copy of the one before it; that copy is addressable here and +// yields a valid path, though its content is not distinct. +// +// Returns ErrContentNotFound if i is outside the range of Leafs. +func (m *MerkleTree) GetMerklePathByIndex(i int) ([][]byte, []int64, error) { + if i < 0 || i >= len(m.Leafs) { + return nil, nil, fmt.Errorf("%w: no leaf at index %d, the tree has %d", ErrContentNotFound, i, len(m.Leafs)) + } + + merklePath, index := m.pathFromLeaf(m.Leafs[i]) + + return merklePath, index, nil +} + +// AppendMerklePath appends the audit path for the leaf holding content to path, and +// the side each sibling sits on to index, returning the extended slices. It is +// GetMerklePath in the style of strconv.AppendInt: the proof produced is identical, +// but where the slices land is the caller's decision. Passing slices with capacity to +// spare - most simply the previous proof's, resliced to length zero - makes proof +// generation allocate nothing, which is worth having where proofs are served at rate: +// the per-proof slice allocations are otherwise the ceiling on concurrent throughput, +// with the garbage collector as the shared resource every goroutine queues on. +// +// var path [][]byte +// var index []int64 +// for _, c := range contents { +// path, index, err = tree.AppendMerklePath(path[:0], index[:0], c) +// ... +// } +// +// The appended hashes are the tree's own, not copies; treat them as read only, and +// note that reusing the path buffer overwrites the positions of the previous proof, +// so a proof that must outlive the next call has to be copied out. +// +// Content is located exactly as GetMerklePath locates it, including the leaf index +// when the tree has one and the first-match rule. If no leaf holds the content, the +// returned error wraps ErrContentNotFound and the slices are returned unchanged. +func (m *MerkleTree) AppendMerklePath(path [][]byte, index []int64, content Content) ([][]byte, []int64, error) { + i, err := m.findLeaf(content) + if err != nil { + return path, index, err + } + if i < 0 { + return path, index, ErrContentNotFound + } + + path, index = m.appendPathFromLeaf(path, index, m.Leafs[i]) + + return path, index, nil +} + +// AppendMerklePathByIndex appends the audit path for the leaf at position i in Leafs +// to path, and the side each sibling sits on to index, returning the extended slices. +// It is to GetMerklePathByIndex what AppendMerklePath is to GetMerklePath, and the +// notes there - identical proofs, reused buffers making generation allocation free, +// appended hashes being the tree's own - apply unchanged. +// +// Returns ErrContentNotFound and the slices unchanged if i is outside the range of +// Leafs. +func (m *MerkleTree) AppendMerklePathByIndex(path [][]byte, index []int64, i int) ([][]byte, []int64, error) { + if i < 0 || i >= len(m.Leafs) { + return path, index, fmt.Errorf("%w: no leaf at index %d, the tree has %d", ErrContentNotFound, i, len(m.Leafs)) + } + + path, index = m.appendPathFromLeaf(path, index, m.Leafs[i]) + + return path, index, nil +} + +// buildWithContent is a helper function that for a given set of Contents, generates a +// corresponding tree and returns the root node, a list of leaf nodes, and a possible error. +// Returns ErrNoContent if cs is empty and ErrNilContent if any entry is nil. +// Chunking limits for a parallel build. Content hashing costs whatever the caller's +// CalculateHash costs, which is unknowable here, so it is spread as thinly as the +// goroutine budget allows. Interior hashing is a fixed, small amount of work per node, +// so a level is only worth splitting up once it is large enough that coordinating the +// split costs less than doing the work in one goroutine. +const ( + minLeafChunk = 1 + minInteriorChunk = 64 + parallelInteriorMinNodes = 1024 +) + +// buildWorkers is the goroutine budget for a build, where one means build serially. +func (m *MerkleTree) buildWorkers() int { + if m.parallelism > 1 { + return m.parallelism + } + + return 1 +} + +// newHashers creates n hashers up front, on the calling goroutine. A parallel build +// hands one to each worker rather than calling hashStrategy from the workers, so a +// caller supplied strategy is never invoked concurrently even though the build is. +func (m *MerkleTree) newHashers(n int) []hash.Hash { + hashers := make([]hash.Hash, n) + for i := range hashers { + hashers[i] = m.hashStrategy() + } + + return hashers +} + +// runParallel splits [0, n) into contiguous chunks across at most workers goroutines and +// calls fn for every index. fn receives the index of the worker running it, so it can +// reach for per-worker state such as that worker's hasher. +// +// The error returned is the one from the lowest failing index rather than from whichever +// goroutine happened to report first, so the same input always fails the same way. +func runParallel(n, workers, minChunk int, fn func(worker, i int) error) error { + chunk := (n + workers - 1) / workers + if chunk < minChunk { + chunk = minChunk + } + + var ( + wg sync.WaitGroup + mu sync.Mutex + firstErr error + errIndex = -1 + panicked any + ) + + worker := 0 + for start := 0; start < n; start += chunk { + end := start + chunk + if end > n { + end = n + } + + wg.Add(1) + go func(w, s, e int) { + defer wg.Done() + // A panic out of Content.CalculateHash would kill the process from here, + // where a serial build would have delivered it to the caller. Carry it + // back and re-raise it on the calling goroutine instead. + defer func() { + if r := recover(); r != nil { + mu.Lock() + if panicked == nil { + panicked = r + } + mu.Unlock() + } + }() + + for i := s; i < e; i++ { + if err := fn(w, i); err != nil { + mu.Lock() + if errIndex == -1 || i < errIndex { + firstErr, errIndex = err, i + } + mu.Unlock() + + return } - current = currentParent - currentParent = currentParent.Parent } - return merklePath, index, nil - } + }(worker, start, end) + worker++ } - return nil, nil, nil + wg.Wait() + + if panicked != nil { + panic(panicked) + } + + return firstErr } -//buildWithContent is a helper function that for a given set of Contents, generates a -//corresponding tree and returns the root node, a list of leaf nodes, and a possible error. -//Returns an error if cs contains no Contents. +// Building allocates the nodes of each level as a single slab rather than one at a +// time, and appends the digests of a level into a single buffer rather than calling +// Sum(nil) per node. The tree that comes out is the same one either way - the same +// Node values, the same Parent, Left, Right and Tree pointers, the same content held +// on the leaves - but the node count is linear in the leaves, so allocating per node +// makes allocation, not hashing, the dominant cost of construction. +// +// Nodes are handed out as pointers into a slab that is sized exactly and never +// appended to, so no reallocation can invalidate a pointer already taken from it. func buildWithContent(cs []Content, t *MerkleTree) (*Node, []*Node, error) { if len(cs) == 0 { - return nil, nil, errors.New("error: cannot construct tree with no content") + return nil, nil, ErrNoContent + } + + // One hasher serves the whole build. It never leaves this call tree, so a tree + // under construction shares no hash state with anything verifying concurrently. + h := t.hashStrategy() + + // A default-construction tree pads an odd leaf count with a duplicate; an + // RFC 6962 tree splits instead and so holds exactly what the caller supplied. + leafCount := len(cs) + if !t.rfc6962 && leafCount%2 == 1 { + leafCount++ + } + slab := make([]Node, leafCount) + leafs := make([]*Node, 0, leafCount) + + // Only RFC 6962 hashes the leaf digest again, so only it needs somewhere to put + // the result. The default construction records what CalculateHash returned. + var ( + leafBuf []byte + leafSize int + ) + if t.rfc6962 { + leafSize = h.Size() + leafBuf = make([]byte, len(cs)*leafSize) } - var leafs []*Node - for _, c := range cs { + + // hashLeafAt fills in leaf i using the given hasher. Every leaf writes only to its + // own slab entry and its own region of leafBuf, so running this across goroutines + // needs no coordination and cannot depend on the order they run in. + hashLeafAt := func(lh hash.Hash, i int) error { + c := cs[i] + // A nil entry would panic on the call below, so reject it as an error the + // caller can handle rather than a fault. + if c == nil { + return fmt.Errorf("%w: index %d", ErrNilContent, i) + } hash, err := c.CalculateHash() if err != nil { + return err + } + if t.rfc6962 { + off := i * leafSize + if hash, err = t.appendLeafDigest(lh, leafBuf[off:off:off+leafSize], hash); err != nil { + return err + } + } + + n := &slab[i] + n.Hash = hash + n.C = c + n.leaf = true + n.Tree = t + + return nil + } + + workers := t.buildWorkers() + if workers > 1 { + // Content hashing is spread across the budget whatever the leaf count, since + // how expensive it is belongs to the caller and cannot be guessed here. + var hashers []hash.Hash + if t.rfc6962 { + hashers = t.newHashers(workers) + } else { + hashers = make([]hash.Hash, workers) + } + if err := runParallel(len(cs), workers, minLeafChunk, func(w, i int) error { + return hashLeafAt(hashers[w], i) + }); err != nil { return nil, nil, err } + } else { + for i := range cs { + if err := hashLeafAt(h, i); err != nil { + return nil, nil, err + } + } + } - leafs = append(leafs, &Node{ - Hash: hash, - C: c, - leaf: true, - Tree: t, - }) - } - if len(leafs)%2 == 1 { - duplicate := &Node{ - Hash: leafs[len(leafs)-1].Hash, - C: leafs[len(leafs)-1].C, - leaf: true, - dup: true, - Tree: t, + for i := range cs { + leafs = append(leafs, &slab[i]) + } + if t.rfc6962 { + // RFC 6962 splits the leaves rather than padding them, so there is no + // duplicate to append and Leafs holds exactly what the caller supplied. + root, err := buildRFC6962(leafs, t, h) + if err != nil { + return nil, nil, err } - leafs = append(leafs, duplicate) + + return root, leafs, nil } - root, err := buildIntermediate(leafs, t) + if len(cs)%2 == 1 { + last := leafs[len(leafs)-1] + n := &slab[len(cs)] + n.Hash = last.Hash + n.C = last.C + n.leaf = true + n.dup = true + n.Tree = t + leafs = append(leafs, n) + } + root, err := buildIntermediate(leafs, t, h) if err != nil { return nil, nil, err } @@ -179,46 +992,271 @@ func buildWithContent(cs []Content, t *MerkleTree) (*Node, []*Node, error) { return root, leafs, nil } -//buildIntermediate is a helper function that for a given list of leaf nodes, constructs -//the intermediate and root levels of the tree. Returns the resulting root node of the tree. -func buildIntermediate(nl []*Node, t *MerkleTree) (*Node, error) { - var nodes []*Node - for i := 0; i < len(nl); i += 2 { - h := t.hashStrategy() - var left, right int = i, i + 1 - if i+1 == len(nl) { - right = i - } - chash := append(nl[left].Hash, nl[right].Hash...) - if _, err := h.Write(chash); err != nil { - return nil, err - } - n := &Node{ - Left: nl[left], - Right: nl[right], - Hash: h.Sum(nil), - Tree: t, +// buildRFC6962 assembles the tree described by RFC 6962 section 2.1. The node list is +// split at the largest power of two below its length, so a count that is not a power +// of two yields an unbalanced tree instead of a duplicated node. A single node is its +// own root, which is why a one item RFC 6962 tree has a root equal to its only leaf +// hash and an empty audit path. +// +// https://datatracker.ietf.org/doc/html/rfc6962#section-2.1 +func buildRFC6962(nl []*Node, t *MerkleTree, h hash.Hash) (*Node, error) { + // Splitting rather than padding means the interior nodes of a tree over n leaves + // number exactly n-1, whatever shape the splits produce, so one slab covers the + // whole recursion. More than that: the subtree over any k consecutive leaves + // owns exactly k-1 of them, so every subtree's slab and digest region is known + // before anything is built - which is what lets the recursion fork. + b := &rfc6962Builder{ + t: t, + size: h.Size(), + slab: make([]Node, len(nl)-1), + buf: make([]byte, (len(nl)-1)*h.Size()), + } + + // The same policy buildIntermediate applies: interior work is small and fixed + // per node, so the tree only earns the goroutines once it has enough of them. + if workers := t.buildWorkers(); workers > 1 && len(nl) >= parallelInteriorMinNodes { + return b.buildParallel(nl, 0, t.newHashers(workers)) + } + + return b.build(nl, 0, h) +} + +// rfc6962Builder carries the slab and the digest buffer down the recursion, so that +// assembling the tree costs two allocations rather than one per node. +// +// Regions are addressed by position rather than handed out from a cursor: the subtree +// over nl owns slab entries and digest slots [base, base+len(nl)-1), its left child +// the first k-1 of them, its right child the next len(nl)-k-1, and the node joining +// them the last one. Sibling regions are disjoint by construction, which is what lets +// buildParallel assemble them on separate goroutines with no coordination at all. +type rfc6962Builder struct { + t *MerkleTree + size int + slab []Node + buf []byte +} + +// build assembles the subtree over nl into region [base, base+len(nl)-1) of the slab +// and digest buffer, reusing h, and returns the subtree's root. +func (b *rfc6962Builder) build(nl []*Node, base int, h hash.Hash) (*Node, error) { + if len(nl) == 1 { + return nl[0], nil + } + + k := largestPowerOfTwoBelow(len(nl)) + left, err := b.build(nl[:k], base, h) + if err != nil { + return nil, err + } + right, err := b.build(nl[k:], base+k-1, h) + if err != nil { + return nil, err + } + + return b.join(left, right, base+len(nl)-2, h) +} + +// join builds the node above left and right in slab slot i. +func (b *rfc6962Builder) join(left, right *Node, i int, h hash.Hash) (*Node, error) { + off := i * b.size + // Cap the slice at its own digest so that appending to a node's Hash cannot + // reach into the digest stored after it. + nodeHash, err := b.t.appendInteriorHash(h, b.buf[off:off:off+b.size], left.Hash, right.Hash) + if err != nil { + return nil, err + } + + n := &b.slab[i] + n.Left = left + n.Right = right + n.Hash = nodeHash + n.Tree = b.t + left.Parent = n + right.Parent = n + + return n, nil +} + +// buildParallel is build forking across goroutines. hashers is the goroutine budget: +// each fork splits it between the two subtrees in proportion to their size, and a +// subtree whose share is down to one hasher, or which is too small to be worth the +// coordination, is built serially with the first hasher of its share. The shares stay +// disjoint the way the slab regions do, and the whole budget is created up front on +// the calling goroutine, so a caller supplied strategy is never invoked concurrently +// even though the build is. +func (b *rfc6962Builder) buildParallel(nl []*Node, base int, hashers []hash.Hash) (*Node, error) { + if len(hashers) == 1 || len(nl) < parallelInteriorMinNodes { + return b.build(nl, base, hashers[0]) + } + + k := largestPowerOfTwoBelow(len(nl)) + share := len(hashers) * k / len(nl) + if share < 1 { + share = 1 + } + if share > len(hashers)-1 { + share = len(hashers) - 1 + } + + var ( + wg sync.WaitGroup + right *Node + rightErr error + panicked any + ) + wg.Add(1) + go func() { + defer wg.Done() + // A panic out of a caller supplied hasher would kill the process from here, + // where a serial build would have delivered it to the caller. Carry it back + // and re-raise it on the calling goroutine instead. + defer func() { + if r := recover(); r != nil { + panicked = r + } + }() + right, rightErr = b.buildParallel(nl[k:], base+k-1, hashers[share:]) + }() + + left, leftErr := b.buildParallel(nl[:k], base, hashers[:share]) + wg.Wait() + + if panicked != nil { + panic(panicked) + } + // The serial build reaches the left subtree's error first; deliver errors in + // the same order however the goroutines happened to finish, so the same input + // always fails the same way. + if leftErr != nil { + return nil, leftErr + } + if rightErr != nil { + return nil, rightErr + } + + return b.join(left, right, base+len(nl)-2, hashers[0]) +} + +// largestPowerOfTwoBelow returns the largest power of two strictly less than n, which +// is the split point RFC 6962 uses. n is always greater than one here. +func largestPowerOfTwoBelow(n int) int { + return 1 << (bits.Len(uint(n-1)) - 1) +} + +// buildIntermediate is a helper function that for a given list of leaf nodes, constructs +// the intermediate and root levels of the tree. Returns the resulting root node of the tree. +func buildIntermediate(nl []*Node, t *MerkleTree, h hash.Hash) (*Node, error) { + workers := t.buildWorkers() + + // The worker hashers are created once and reused for every level large enough to + // build in parallel, rather than a fresh set per level. Created lazily, so a + // serial build or a small tree never pays for them. + var hashers []hash.Hash + + for len(nl) > 1 { + // Each pass consumes a level and produces the one above it, so the node + // count is known before any of them are built. + count := (len(nl) + 1) / 2 + size := h.Size() + slab := make([]Node, count) + next := make([]*Node, count) + buf := make([]byte, count*size) + + // buildNode assembles the node above pair p. Pair p reads only nodes 2p and + // 2p+1 and writes only its own slab entry, digest region and next slot, so + // pairs are independent of each other and of the order they are built in. + buildNode := func(nh hash.Hash, p int) error { + left, right := p*2, p*2+1 + if right == len(nl) { + right = left + } + + off := p * size + // Cap the slice at its own digest so that appending to a node's Hash + // cannot reach into the digest stored after it. + nodeHash, err := t.appendInteriorHash(nh, buf[off:off:off+size], nl[left].Hash, nl[right].Hash) + if err != nil { + return err + } + + n := &slab[p] + n.Left = nl[left] + n.Right = nl[right] + n.Hash = nodeHash + n.Tree = t + nl[left].Parent = n + nl[right].Parent = n + next[p] = n + + return nil } - nodes = append(nodes, n) - nl[left].Parent = n - nl[right].Parent = n - if len(nl) == 2 { - return n, nil + + // Interior work is small and fixed per node, so a level only earns the + // coordination once there are enough nodes in it. + if workers > 1 && count >= parallelInteriorMinNodes { + if hashers == nil { + hashers = t.newHashers(workers) + } + if err := runParallel(count, workers, minInteriorChunk, func(w, p int) error { + return buildNode(hashers[w], p) + }); err != nil { + return nil, err + } + } else { + for p := 0; p < count; p++ { + if err := buildNode(h, p); err != nil { + return nil, err + } + } } + nl = next } - return buildIntermediate(nodes, t) + + return nl[0], nil } -//MerkleRoot returns the unverified Merkle Root (hash of the root node) of the tree. +// MerkleRoot returns the unverified Merkle Root (hash of the root node) of the tree. +// +// The returned slice is the tree's own, not a copy; treat it as read only. func (m *MerkleTree) MerkleRoot() []byte { return m.merkleRoot } -//RebuildTree is a helper function that will rebuild the tree reusing only the content that -//it holds in the leaves. +// Sorted reports whether the tree orders each pair of siblings before hashing them, +// as chosen by NewTreeWithHashStrategySorted. +// +// Sorting siblings makes the root independent of the order the content was supplied +// in: [A B C D] and [B A C D] and [D C B A] all produce the same root, because each +// pair is ordered before it is hashed and the pairing itself is all that survives. +// Only regrouping which items are paired together changes the root. Callers that +// depend on the root committing to leaf order should check this is false. +func (m *MerkleTree) Sorted() bool { + return m.sort +} + +// RFC6962 reports whether the tree was built with WithRFC6962, meaning leaf and +// interior hashes carry distinct prefixes and odd node counts are split rather than +// duplicated. +func (m *MerkleTree) RFC6962() bool { + return m.rfc6962 +} + +// RebuildTree is a helper function that will rebuild the tree reusing only the content that +// it holds in the leaves. func (m *MerkleTree) RebuildTree() error { - var cs []Content + // Sized to the leaf count up front; at most one entry, the padding copy, goes + // unused. + cs := make([]Content, 0, len(m.Leafs)) for _, c := range m.Leafs { + // Leafs holds the padding copy that buildWithContent appends when the + // content count is odd. Feeding it back in would promote that copy to + // real content, so the tree would lose track of which leaf is padding + // and report one more item than the caller supplied. The root is + // unaffected either way; skipping it keeps Leafs and the dup marker + // accurate across repeated rebuilds. + if c.dup { + continue + } cs = append(cs, c.C) } root, leafs, err := buildWithContent(cs, m) @@ -228,12 +1266,13 @@ func (m *MerkleTree) RebuildTree() error { m.Root = root m.Leafs = leafs m.merkleRoot = root.Hash + m.buildLeafIndex() return nil } -//RebuildTreeWith replaces the content of the tree and does a complete rebuild; while the root of -//the tree will be replaced the MerkleTree completely survives this operation. Returns an error if the -//list of content cs contains no entries. +// RebuildTreeWith replaces the content of the tree and does a complete rebuild; while the root of +// the tree will be replaced the MerkleTree completely survives this operation. Returns an error if the +// list of content cs contains no entries. func (m *MerkleTree) RebuildTreeWith(cs []Content) error { root, leafs, err := buildWithContent(cs, m) if err != nil { @@ -242,73 +1281,110 @@ func (m *MerkleTree) RebuildTreeWith(cs []Content) error { m.Root = root m.Leafs = leafs m.merkleRoot = root.Hash + m.buildLeafIndex() return nil } -//VerifyTree verify tree validates the hashes at each level of the tree and returns true if the -//resulting hash at the root of the tree matches the resulting root hash; returns false otherwise. +// VerifyTree verify tree validates the hashes at each level of the tree and returns true if the +// resulting hash at the root of the tree matches the resulting root hash; returns false otherwise. func (m *MerkleTree) VerifyTree() (bool, error) { - calculatedMerkleRoot, err := m.Root.verifyNode() + // A zero value MerkleTree has no root to walk. Report it rather than faulting, + // so that a caller handed a tree from elsewhere can tell "never built" apart + // from "built and does not verify". + if m.Root == nil { + return false, fmt.Errorf("%w: tree has no root", ErrMalformedTree) + } + // One hasher and one buffer serve the whole walk. The buffer holds only the path + // being walked, so the depth of the tree is enough for it; append covers the case + // of a Content.CalculateHash that returns a digest wider than the tree's own. + h := m.hashStrategy() + scratch := make([]byte, 0, (bits.Len(uint(len(m.Leafs)))+2)*h.Size()) + + calculatedMerkleRoot, matched, err := m.Root.verifyNode(h, scratch) if err != nil { return false, err } - - if bytes.Compare(m.merkleRoot, calculatedMerkleRoot) == 0 { - return true, nil + if !matched { + return false, nil } - return false, nil + + return bytes.Equal(m.merkleRoot, calculatedMerkleRoot), nil } -//VerifyContent indicates whether a given content is in the tree and the hashes are valid for that content. -//Returns true if the expected Merkle Root is equivalent to the Merkle root calculated on the critical path -//for a given content. Returns true if valid and false otherwise. +// VerifyContent indicates whether a given content is in the tree and the hashes are valid for that content. +// Returns true if the expected Merkle Root is equivalent to the Merkle root calculated on the critical path +// for a given content. Returns true if valid and false otherwise. func (m *MerkleTree) VerifyContent(content Content) (bool, error) { - for _, l := range m.Leafs { - ok, err := l.C.Equals(content) + // Locating the content carries the same O(n) scan as GetMerklePath unless the + // tree was built with WithLeafIndex. + i, err := m.findLeaf(content) + if err != nil { + return false, err + } + if i < 0 { + return false, nil + } + + // One hasher and one buffer serve the whole climb, the way verifyNode threads + // them through its walk. Recomputing a level takes three digests - each child + // from what sits beneath it, then the parent from the pair - and creating a + // hasher and a digest for each made allocation, not hashing, the dominant cost + // here. The buffer never holds more than one level, so three digests is its + // whole working set; append covers a Content.CalculateHash wider than the + // tree's own digest. + h := m.hashStrategy() + scratch := make([]byte, 0, 4*h.Size()) + + for currentParent := m.Leafs[i].Parent; currentParent != nil; currentParent = currentParent.Parent { + if currentParent.Left == nil || currentParent.Right == nil { + return false, fmt.Errorf("%w: interior node is missing a child", ErrMalformedTree) + } + scratch = scratch[:0] + scratch, err = currentParent.Right.appendCalculatedHash(h, scratch) if err != nil { return false, err } + mid := len(scratch) - if ok { - currentParent := l.Parent - for currentParent != nil { - h := m.hashStrategy() - rightBytes, err := currentParent.Right.calculateNodeHash() - if err != nil { - return false, err - } - - leftBytes, err := currentParent.Left.calculateNodeHash() - if err != nil { - return false, err - } + scratch, err = currentParent.Left.appendCalculatedHash(h, scratch) + if err != nil { + return false, err + } + end := len(scratch) - if _, err := h.Write(append(leftBytes, rightBytes...)); err != nil { - return false, err - } - if bytes.Compare(h.Sum(nil), currentParent.Hash) != 0 { - return false, nil - } - currentParent = currentParent.Parent - } - return true, nil + // The children's digests are read into the hasher before Sum appends, so a + // growth of scratch here cannot invalidate them. + scratch, err = m.appendInteriorHash(h, scratch, scratch[mid:end], scratch[:mid]) + if err != nil { + return false, err + } + if !bytes.Equal(scratch[end:], currentParent.Hash) { + return false, nil } } - return false, nil + // The walk above recomputes and checks every hash on the path from the leaf up to + // and including the root node. Confirm that node really is the root this tree + // advertises, otherwise a tampered merkleRoot would go unnoticed. + if m.Root == nil { + return false, fmt.Errorf("%w: tree has no root", ErrMalformedTree) + } + + return bytes.Equal(m.Root.Hash, m.merkleRoot), nil } -//String returns a string representation of the node. +// String returns a string representation of the node. func (n *Node) String() string { return fmt.Sprintf("%t %t %v %s", n.leaf, n.dup, n.Hash, n.C) } -//String returns a string representation of the tree. Only leaf nodes are included -//in the output. +// String returns a string representation of the tree. Only leaf nodes are included +// in the output. func (m *MerkleTree) String() string { - s := "" + // One builder rather than repeated string concatenation, which re-copies the + // whole prefix on every leaf and turns printing a large tree quadratic. + var sb strings.Builder for _, l := range m.Leafs { - s += fmt.Sprint(l) - s += "\n" + fmt.Fprintln(&sb, l) } - return s + return sb.String() } diff --git a/vendor/github.com/cbergoon/merkletree/proof.go b/vendor/github.com/cbergoon/merkletree/proof.go new file mode 100644 index 00000000..28c0b522 --- /dev/null +++ b/vendor/github.com/cbergoon/merkletree/proof.go @@ -0,0 +1,230 @@ +// Copyright 2017 Cameron Bergoon +// Licensed under the MIT License, see LICENCE file for details. + +package merkletree + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "hash" + "sync" +) + +// ErrMalformedProof is returned when a proof cannot be walked at all: a path and an +// index of different lengths, or a side marker that is neither 0 nor 1. It reports a +// proof that is the wrong shape rather than one that is the right shape and does not +// verify, which is an ordinary false. Test for it with errors.Is. +var ErrMalformedProof = errors.New("error: proof is malformed") + +// proofScratch is the working state one proof verification needs: a hasher and the +// buffer the replay slides its digests through. Pooled as a pair, so the option-less +// verification path allocates nothing of its own. +type proofScratch struct { + h hash.Hash + buf []byte +} + +// defaultProofConfig is the configuration an option-less VerifyProof runs under: the +// same value configFromOptions produces for an empty option list, allocated once +// rather than per verification. It is shared across goroutines and must never be +// mutated; the verification path only ever reads it. +// +// Being private and immutable is also what makes it the one place a scratch pool is +// safe; see the field's comment on MerkleTree. +var defaultProofConfig = &MerkleTree{ + hashStrategy: sha256.New, + scratchPool: &sync.Pool{New: func() any { + return &proofScratch{h: sha256.New(), buf: make([]byte, 0, 2*sha256.Size)} + }}, +} + +// VerifyProof reports whether content, the audit path and the side index returned by +// GetMerklePath together reproduce root. +// +// This is the operation a verifier performs when it holds a root obtained from somewhere +// it trusts, and a proof obtained from somewhere it does not. It needs no tree: the +// whole point is that the party checking the proof does not have to hold, rebuild, or +// have ever seen the tree the proof came from. MerkleTree.VerifyContent answers a +// different question, on a tree it already has. +// +// The options configure the construction the proof was produced under, and they must +// match the tree that produced it or verification fails. Only the options that affect +// hashing are meaningful here: WithHasher, WithSortedSiblings and WithRFC6962. The +// others describe how a tree is built rather than how its hashes are computed, and are +// accepted and ignored so that the same option list can be shared with the constructor. +// A tree's own settings are readable through MerkleTree.Sorted and MerkleTree.RFC6962. +// +// Returns false when the proof simply does not reproduce root, and an error when the +// proof is malformed or hashing the content fails. +// +// # What a verified proof does and does not establish +// +// It establishes that content sits in some tree whose root is root. It says nothing +// about whether root is the root a verifier should be trusting; that has to arrive +// through a channel the verifier already trusts, and this function cannot check it. +// +// Under the default construction it establishes less than it appears to. Leaf and +// interior hashes are computed the same way, so nothing distinguishes a leaf digest from +// an interior one, and a path can be presented that treats an interior node as though it +// were a leaf. Building the tree with WithRFC6962 closes this: the two kinds of hash +// carry distinct prefixes, so a forgery of that shape would need a genuine collision. +// The distinction matters far more here than it does for VerifyContent, which walks a +// real tree and so can only be shown nodes that are really in it. Prefer WithRFC6962 for +// proofs that arrive from somewhere untrusted. +func VerifyProof(content Content, path [][]byte, index []int64, root []byte, opts ...TreeOption) (bool, error) { + if content == nil { + return false, ErrNilContent + } + digest, err := content.CalculateHash() + if err != nil { + return false, err + } + + return VerifyProofWithDigest(digest, path, index, root, opts...) +} + +// VerifyProofWithDigest is VerifyProof for a verifier that holds the leaf digest rather +// than the content itself, and so has no reason to implement Content. +// +// The digest is the value Content.CalculateHash returns for the leaf, which is the leaf +// hash the tree records under the default construction. Under WithRFC6962 the tree +// records the prefixed hash of that digest instead, and this function applies the prefix +// itself, so the argument is the same value either way: what CalculateHash returned, not +// what the tree stored. +func VerifyProofWithDigest(digest []byte, path [][]byte, index []int64, root []byte, opts ...TreeOption) (bool, error) { + if len(path) != len(index) { + return false, fmt.Errorf("%w: path has %d entries and the index has %d", ErrMalformedProof, len(path), len(index)) + } + + // With no options there is nothing to resolve, so the shared default carries the + // configuration instead of a fresh one allocated per call. rootFromProof only + // reads it, which is what makes sharing it across goroutines safe. + cfg := defaultProofConfig + if len(opts) > 0 { + var err error + if cfg, err = configFromOptions(opts); err != nil { + return false, err + } + } + + return cfg.proofReproducesRoot(digest, path, index, root) +} + +// proofReproducesRoot replays a proof from the leaf upwards and reports whether it +// arrives at root. +// +// The hashing goes through the same appendLeafDigest and appendInteriorHash the +// constructor uses, so a proof is checked with the code that built the tree rather than +// with a second implementation of the same rules that could drift from it. What keeps +// that from being circular is the oracle module, which checks the construction itself +// against the RFC 6962 reference implementation. +// +// Merkle roots are public values, so the ordinary comparison at the end is +// appropriate; there is no secret here whose length or content a timing difference +// could leak. +func (m *MerkleTree) proofReproducesRoot(digest []byte, path [][]byte, index []int64, root []byte) (bool, error) { + // One hasher and one buffer serve the whole replay, recycled through the pool + // when the configuration carries one. Each level appends its hash and then + // slides it down over the previous one, so the buffer stays the width of a + // single digest however tall the tree is. Doing the comparison here rather than + // returning the computed root is what lets the buffer go back to the pool. + var ( + scratch *proofScratch + h hash.Hash + buf []byte + ) + if m.scratchPool != nil { + scratch = m.scratchPool.Get().(*proofScratch) + h, buf = scratch.h, scratch.buf[:0] + } else { + h = m.hashStrategy() + buf = make([]byte, 0, 2*h.Size()) + } + defer func() { + if scratch != nil { + scratch.buf = buf + m.scratchPool.Put(scratch) + } + }() + + var err error + if m.rfc6962 { + if buf, err = m.appendLeafDigest(h, buf, digest); err != nil { + return false, err + } + } else { + buf = append(buf, digest...) + } + + for k := range path { + // GetMerklePath records 1 when the sibling is the right hand node, which is + // to say when the node being carried up is the left hand one. + var left, right []byte + switch index[k] { + case 1: + left, right = buf, path[k] + case 0: + left, right = path[k], buf + default: + return false, fmt.Errorf("%w: index entry %d is %d, expected 0 or 1", ErrMalformedProof, k, index[k]) + } + + off := len(buf) + if buf, err = m.appendInteriorHash(h, buf, left, right); err != nil { + return false, err + } + buf = buf[:copy(buf, buf[off:])] + } + + return bytes.Equal(buf, root), nil +} + +// configFromOptions applies opts to a MerkleTree that carries configuration and nothing +// else, and reports the same conflicts the constructor does. +// +// The verification path and the construction path resolve options through this one +// function, so a tree and a proof checked against it can never disagree about what an +// option meant. +func configFromOptions(opts []TreeOption) (*MerkleTree, error) { + m := &MerkleTree{ + hashStrategy: sha256.New, + } + for _, opt := range opts { + if opt == nil { + continue + } + opt(m) + } + if m.hashStrategy == nil { + return nil, errors.New("error: hash strategy cannot be nil") + } + if m.rfc6962 && m.sort { + return nil, errors.New("error: WithRFC6962 and WithSortedSiblings cannot be combined; RFC 6962 specifies its own sibling ordering") + } + + return m, nil +} + +// VerifyProof reports whether the given proof reproduces this tree's root, using this +// tree's construction settings. +// +// It is the convenience form of the package level VerifyProof for a caller that does +// hold the tree, and saves restating the tree's options. It still verifies the proof it +// is given rather than looking anything up, so it will reject a proof for content this +// tree holds if that proof was generated somewhere else under different settings. +func (m *MerkleTree) VerifyProof(content Content, path [][]byte, index []int64) (bool, error) { + if content == nil { + return false, ErrNilContent + } + if len(path) != len(index) { + return false, fmt.Errorf("%w: path has %d entries and the index has %d", ErrMalformedProof, len(path), len(index)) + } + digest, err := content.CalculateHash() + if err != nil { + return false, err + } + + return m.proofReproducesRoot(digest, path, index, m.merkleRoot) +} diff --git a/vendor/github.com/cbergoon/merkletree/serialize.go b/vendor/github.com/cbergoon/merkletree/serialize.go new file mode 100644 index 00000000..d1cdba2a --- /dev/null +++ b/vendor/github.com/cbergoon/merkletree/serialize.go @@ -0,0 +1,913 @@ +// Copyright 2017 Cameron Bergoon +// Licensed under the MIT License, see LICENCE file for details. + +package merkletree + +import ( + "bytes" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "reflect" + "slices" + "sync" +) + +// A serialized tree records only what is needed to regenerate it: the ordered leaf +// content, the name of the hash strategy, the sibling sort flag, and the Merkle root +// the encoder observed. Every other part of the tree - the intermediate nodes, the +// parent and tree back-pointers, the leaf and padding markers - is derived from those +// inputs by buildWithContent, so storing it would be redundant. +// +// Persisting the derived structure is not merely wasteful, it is impossible with the +// standard codecs: Node points at its Tree and its Parent while the tree points back +// down at its Root and Leafs, and those cycles send encoding/gob and encoding/json +// into unbounded recursion rather than an error. Recording the seed sidesteps the +// cycle instead of working around it, and the recorded root turns decoding into an +// integrity check for free - a corrupted payload, a mismatched hash strategy, or a +// non-deterministic content encoder all surface as a root mismatch. +const ( + // serializationMagic prefixes every binary blob so a truncated or foreign + // payload is rejected before any length is trusted. + serializationMagic = "MTREE" + // serializationVersion is the wire format version written by this package. + // Version 2 added the RFC 6962 flag after the sort flag. + serializationVersion = 2 +) + +var ( + // ErrUnsupportedVersion is returned when decoding a tree written by a newer + // format version than this build understands. + ErrUnsupportedVersion = errors.New("merkletree: unsupported serialization version") + // ErrCorruptData is returned when a serialized tree is malformed, truncated, + // or not a merkletree payload at all. + ErrCorruptData = errors.New("merkletree: corrupt serialized tree") + // ErrRootMismatch is returned when the tree rebuilt from a payload does not + // hash to the Merkle root recorded in that payload. + ErrRootMismatch = errors.New("merkletree: rebuilt Merkle root does not match the encoded root") + // ErrNoHashStrategy is returned when a hash strategy cannot be named at encode + // time or resolved at decode time. + ErrNoHashStrategy = errors.New("merkletree: unknown hash strategy") + // ErrNoContentType is returned when a content type cannot be named at encode + // time or resolved at decode time. + ErrNoContentType = errors.New("merkletree: unregistered content type") +) + +// hashStrategyRegistry maps names to hash strategies and back. A function value +// cannot be serialized, so a tree records the name of its strategy and the decoder +// looks the name up here. The reverse index is keyed by the strategy's code pointer, +// which is stable for both plain functions and closure literals. +var hashStrategyRegistry = struct { + sync.RWMutex + byName map[string]func() hash.Hash + byFunc map[uintptr]string +}{ + byName: map[string]func() hash.Hash{}, + byFunc: map[uintptr]string{}, +} + +func init() { + RegisterHashStrategy("sha256", sha256.New) + RegisterHashStrategy("sha224", sha256.New224) + RegisterHashStrategy("sha512", sha512.New) + RegisterHashStrategy("sha384", sha512.New384) + RegisterHashStrategy("sha512_224", sha512.New512_224) + RegisterHashStrategy("sha512_256", sha512.New512_256) + RegisterHashStrategy("sha1", sha1.New) + RegisterHashStrategy("md5", md5.New) +} + +// RegisterHashStrategy makes a hash strategy serializable under the given name. The +// hash strategies in the standard library are registered automatically; anything else +// - keccak256, blake2b, or a strategy of your own - must be registered before a tree +// using it can be marshaled or unmarshaled. +// +// merkletree.RegisterHashStrategy("keccak256", sha3.NewLegacyKeccak256) +// t, err := merkletree.NewTreeWithHashStrategy(list, sha3.NewLegacyKeccak256) +// +// Registration is idempotent: registering the same name and strategy again is a no-op. +// Registering a different strategy under a name already in use panics, because doing +// so would silently change the meaning of previously written payloads. Registering a +// second name for an already registered strategy is allowed, but the first name wins +// when encoding. +// +// RegisterHashStrategy is safe for concurrent use, though the natural place to call +// it is package initialization. +func RegisterHashStrategy(name string, strategy func() hash.Hash) { + if name == "" { + panic("merkletree: cannot register a hash strategy under an empty name") + } + if strategy == nil { + panic("merkletree: cannot register a nil hash strategy") + } + + ptr := reflect.ValueOf(strategy).Pointer() + + hashStrategyRegistry.Lock() + defer hashStrategyRegistry.Unlock() + + if existing, ok := hashStrategyRegistry.byName[name]; ok { + if reflect.ValueOf(existing).Pointer() == ptr { + return + } + panic(fmt.Sprintf("merkletree: hash strategy %q is already registered to a different function", name)) + } + hashStrategyRegistry.byName[name] = strategy + if _, ok := hashStrategyRegistry.byFunc[ptr]; !ok { + hashStrategyRegistry.byFunc[ptr] = name + } +} + +// HashStrategyNames returns the sorted names of every registered hash strategy. +func HashStrategyNames() []string { + hashStrategyRegistry.RLock() + defer hashStrategyRegistry.RUnlock() + + names := make([]string, 0, len(hashStrategyRegistry.byName)) + for name := range hashStrategyRegistry.byName { + names = append(names, name) + } + slices.Sort(names) + return names +} + +// lookupHashStrategy resolves a registered name to its strategy. +func lookupHashStrategy(name string) (func() hash.Hash, bool) { + hashStrategyRegistry.RLock() + defer hashStrategyRegistry.RUnlock() + + strategy, ok := hashStrategyRegistry.byName[name] + return strategy, ok +} + +// lookupHashStrategyName resolves a strategy back to the name it was registered under. +func lookupHashStrategyName(strategy func() hash.Hash) (string, bool) { + if strategy == nil { + return "", false + } + + hashStrategyRegistry.RLock() + defer hashStrategyRegistry.RUnlock() + + name, ok := hashStrategyRegistry.byFunc[reflect.ValueOf(strategy).Pointer()] + return name, ok +} + +// contentRegistry maps names to content types and back, mirroring the role gob.Register +// plays for interface values. Content is defined by the application, so the tree cannot +// know how to rebuild a concrete type from bytes without being told. +var contentRegistry = struct { + sync.RWMutex + byName map[string]reflect.Type + byType map[reflect.Type]string +}{ + byName: map[string]reflect.Type{}, + byType: map[reflect.Type]string{}, +} + +// RegisterContent makes a Content type serializable by the registry-backed marshalers +// (MarshalBinary, MarshalJSON, and anything built on them such as encoding/gob), under +// a name derived from its package path and type name. +// +// The value passed is a template, not data: pass a zero value of the type exactly as it +// is stored in the tree. Registering TestContent{} round-trips leaves back to +// TestContent values; registering &TestContent{} round-trips them back to pointers. +// +// The type must marshal itself. c must implement encoding.BinaryMarshaler and a pointer +// to it must implement encoding.BinaryUnmarshaler, which is the usual pairing of a value +// receiver for MarshalBinary and a pointer receiver for UnmarshalBinary. RegisterContent +// panics if either is missing, or if the type is unnamed. +// +// Callers who would rather not use a package-level registry can skip registration +// entirely and use MarshalWith and UnmarshalWith instead. +func RegisterContent(c Content) { + rt := reflect.TypeOf(c) + if rt == nil { + panic("merkletree: cannot register nil content") + } + name := contentTypeName(rt) + if name == "" { + panic(fmt.Sprintf("merkletree: cannot register unnamed content type %s", rt)) + } + RegisterContentName(name, c) +} + +// RegisterContentName is RegisterContent with an explicit name. Use it to pin a stable +// name into the wire format so that renaming or moving the Go type does not invalidate +// payloads already written. +func RegisterContentName(name string, c Content) { + if name == "" { + panic("merkletree: cannot register content under an empty name") + } + rt := reflect.TypeOf(c) + if rt == nil { + panic("merkletree: cannot register nil content") + } + if _, ok := c.(encoding.BinaryMarshaler); !ok { + panic(fmt.Sprintf("merkletree: content type %s does not implement encoding.BinaryMarshaler", rt)) + } + // A pointer to the underlying type must be able to load itself from bytes. When a + // pointer type is registered that is rt itself; when a value type is registered it + // is the address of a fresh value, which is what the decoder will unmarshal into. + ptrType := rt + if ptrType.Kind() != reflect.Pointer { + ptrType = reflect.PointerTo(rt) + } + if !ptrType.Implements(reflect.TypeOf((*encoding.BinaryUnmarshaler)(nil)).Elem()) { + panic(fmt.Sprintf("merkletree: %s does not implement encoding.BinaryUnmarshaler", ptrType)) + } + + contentRegistry.Lock() + defer contentRegistry.Unlock() + + if existing, ok := contentRegistry.byName[name]; ok { + if existing == rt { + return + } + panic(fmt.Sprintf("merkletree: content name %q is already registered to %s", name, existing)) + } + contentRegistry.byName[name] = rt + if _, ok := contentRegistry.byType[rt]; !ok { + contentRegistry.byType[rt] = name + } +} + +// contentTypeName derives a stable, unambiguous name for a content type. Pointer depth +// is preserved so that T and *T round-trip back to what was registered. +func contentTypeName(rt reflect.Type) string { + prefix := "" + for rt.Kind() == reflect.Pointer { + prefix += "*" + rt = rt.Elem() + } + if rt.Name() == "" { + return "" + } + if rt.PkgPath() == "" { + return prefix + rt.Name() + } + return prefix + rt.PkgPath() + "." + rt.Name() +} + +// contentTypeCache remembers the last content type resolved against a registry. A +// tree overwhelmingly holds one concrete type, so with the cache the registry lock +// and lookup are paid once per encode or decode rather than once per leaf. The zero +// value is an empty cache; it is single-use state for one loop, never shared. +type contentTypeCache struct { + name string + rt reflect.Type +} + +// marshalRegisteredContent encodes a single content item along with the name needed to +// rebuild its concrete type. +func marshalRegisteredContent(c Content, cache *contentTypeCache) (string, []byte, error) { + rt := reflect.TypeOf(c) + if rt == nil { + return "", nil, fmt.Errorf("%w: tree contains nil content", ErrNoContentType) + } + + name := cache.name + if rt != cache.rt || cache.name == "" { + contentRegistry.RLock() + var ok bool + name, ok = contentRegistry.byType[rt] + contentRegistry.RUnlock() + + if !ok { + return "", nil, fmt.Errorf("%w: %s; call merkletree.RegisterContent to make it serializable, or use MarshalWith", ErrNoContentType, rt) + } + cache.name, cache.rt = name, rt + } + + bm, ok := c.(encoding.BinaryMarshaler) + if !ok { + return "", nil, fmt.Errorf("%w: %s does not implement encoding.BinaryMarshaler", ErrNoContentType, rt) + } + payload, err := bm.MarshalBinary() + if err != nil { + return "", nil, fmt.Errorf("merkletree: marshaling content of type %s: %w", rt, err) + } + return name, payload, nil +} + +// unmarshalRegisteredContent rebuilds a concrete content value from its recorded name +// and payload. The cache carries the last resolved name across the records of one +// decode; the binary decoder interns repeated names, so the comparison is usually a +// pointer match. +func unmarshalRegisteredContent(name string, payload []byte, cache *contentTypeCache) (Content, error) { + if name == "" { + return nil, fmt.Errorf("%w: payload carries no content type names, which means it was written by MarshalWith; decode it with UnmarshalWith", ErrNoContentType) + } + + rt := cache.rt + if name != cache.name || rt == nil { + contentRegistry.RLock() + var ok bool + rt, ok = contentRegistry.byName[name] + contentRegistry.RUnlock() + + if !ok { + return nil, fmt.Errorf("%w: %q; call merkletree.RegisterContent for that type before unmarshaling", ErrNoContentType, name) + } + cache.name, cache.rt = name, rt + } + + isPointer := rt.Kind() == reflect.Pointer + elem := rt + if isPointer { + elem = rt.Elem() + } + + pv := reflect.New(elem) + bu, ok := pv.Interface().(encoding.BinaryUnmarshaler) + if !ok { + return nil, fmt.Errorf("%w: %s does not implement encoding.BinaryUnmarshaler", ErrNoContentType, pv.Type()) + } + // UnmarshalBinary implementations are permitted to retain the slice they are + // handed, so payload has to be one no one else holds. Both decoders already + // produce exactly that: the binary reader carves each record its own capped + // region of a decode-wide arena, and encoding/json decodes base64 into a fresh + // slice. The treeData carrying them is transient and is dropped once the tree is + // built, so there is nothing left to alias. Copying again here duplicated every + // content payload on the way in, which on a large tree is one wasted allocation + // per leaf. The UnmarshalWith path has always handed record.Payload straight to + // the caller's decoder on the same reasoning. + if err := bu.UnmarshalBinary(payload); err != nil { + return nil, fmt.Errorf("merkletree: unmarshaling content of type %q: %w", name, err) + } + + var v any = pv.Elem().Interface() + if isPointer { + v = pv.Interface() + } + c, ok := v.(Content) + if !ok { + return nil, fmt.Errorf("%w: %q does not implement merkletree.Content", ErrNoContentType, name) + } + return c, nil +} + +// ContentMarshalFunc encodes a single content item to bytes. It must be deterministic: +// two content items that are Equals must encode identically, or a decoded tree will not +// reproduce the root it was encoded with. +type ContentMarshalFunc func(Content) ([]byte, error) + +// ContentUnmarshalFunc decodes a single content item produced by a ContentMarshalFunc. +type ContentUnmarshalFunc func([]byte) (Content, error) + +// MarshalOption adjusts how a tree is encoded. +type MarshalOption func(*marshalConfig) + +type marshalConfig struct { + hashStrategyName string +} + +// WithHashStrategyName records the given name for the tree's hash strategy instead of +// looking the strategy up in the package registry. Pair it with WithHashStrategy at +// decode time to serialize trees without touching global state at all. +func WithHashStrategyName(name string) MarshalOption { + return func(c *marshalConfig) { c.hashStrategyName = name } +} + +// UnmarshalOption adjusts how a tree is decoded. +type UnmarshalOption func(*unmarshalConfig) + +type unmarshalConfig struct { + hashStrategy func() hash.Hash +} + +// WithHashStrategy rebuilds the tree with the given hash strategy instead of resolving +// the name recorded in the payload through the package registry. The recorded name is +// still preserved, so re-encoding the decoded tree reproduces the original name. +func WithHashStrategy(strategy func() hash.Hash) UnmarshalOption { + return func(c *unmarshalConfig) { c.hashStrategy = strategy } +} + +// treeData is the on-the-wire form of a tree: the seed it can be rebuilt from. +type treeData struct { + Version int `json:"version"` + HashStrategy string `json:"hashStrategy"` + Sort bool `json:"sort"` + RFC6962 bool `json:"rfc6962,omitempty"` + MerkleRoot []byte `json:"merkleRoot"` + Contents []contentRecord `json:"contents"` +} + +// contentRecord is one leaf's content. Type is empty for payloads written by +// MarshalWith, where the caller's own decoder supplies the concrete type. +type contentRecord struct { + Type string `json:"type,omitempty"` + Payload []byte `json:"payload"` +} + +// snapshot captures the tree as the seed it can be rebuilt from. enc may be nil, in +// which case content is encoded through the package registry. +// +// The marshalers all take a value receiver, following the example of time.Time. A +// pointer receiver would leave a tree that is held by value rather than by pointer - +// a struct field, say - falling back to reflection inside encoding/json and +// encoding/gob, which is precisely the unbounded recursion these methods exist to +// prevent. Marshaling only reads, so the copy costs nothing that matters. +func (m MerkleTree) snapshot(enc ContentMarshalFunc, opts ...MarshalOption) (*treeData, error) { + if m.Root == nil || len(m.Leafs) == 0 { + return nil, errors.New("merkletree: cannot marshal an empty tree") + } + + var cfg marshalConfig + for _, opt := range opts { + opt(&cfg) + } + + name := cfg.hashStrategyName + if name == "" { + name = m.hashStrategyName + } + if name == "" { + var ok bool + if name, ok = lookupHashStrategyName(m.hashStrategy); !ok { + return nil, fmt.Errorf("%w: the tree's hash strategy has no registered name; call merkletree.RegisterHashStrategy for it, or pass merkletree.WithHashStrategyName", ErrNoHashStrategy) + } + } + + td := &treeData{ + Version: serializationVersion, + HashStrategy: name, + Sort: m.sort, + RFC6962: m.rfc6962, + MerkleRoot: bytes.Clone(m.merkleRoot), + // Sized to the leaf count up front; at most one entry, the padding copy, goes + // unused, where growing by append reallocates log n times on a large tree. + Contents: make([]contentRecord, 0, len(m.Leafs)), + } + + var cache contentTypeCache + for _, l := range m.Leafs { + // Skip the padding copy buildWithContent appends for an odd content count. + // It is regenerated on rebuild; encoding it would promote it to real content + // and the decoded tree would report one more item than was put in. + if l.dup { + continue + } + if enc != nil { + payload, err := enc(l.C) + if err != nil { + return nil, fmt.Errorf("merkletree: marshaling content: %w", err) + } + td.Contents = append(td.Contents, contentRecord{Payload: payload}) + continue + } + typeName, payload, err := marshalRegisteredContent(l.C, &cache) + if err != nil { + return nil, err + } + td.Contents = append(td.Contents, contentRecord{Type: typeName, Payload: payload}) + } + + if len(td.Contents) == 0 { + return nil, errors.New("merkletree: cannot marshal a tree with no content") + } + return td, nil +} + +// tree rebuilds a tree from its seed and verifies that it hashes back to the recorded +// root. dec may be nil, in which case content is decoded through the package registry. +func (td *treeData) tree(dec ContentUnmarshalFunc, opts ...UnmarshalOption) (*MerkleTree, error) { + if td.Version != serializationVersion { + return nil, fmt.Errorf("%w: got %d, this build writes and reads %d", ErrUnsupportedVersion, td.Version, serializationVersion) + } + + var cfg unmarshalConfig + for _, opt := range opts { + opt(&cfg) + } + + strategy := cfg.hashStrategy + if strategy == nil { + var ok bool + if strategy, ok = lookupHashStrategy(td.HashStrategy); !ok { + return nil, fmt.Errorf("%w: %q; call merkletree.RegisterHashStrategy for it before unmarshaling, or pass merkletree.WithHashStrategy", ErrNoHashStrategy, td.HashStrategy) + } + } + + if len(td.Contents) == 0 { + return nil, errors.New("merkletree: serialized tree contains no content") + } + + cs := make([]Content, 0, len(td.Contents)) + var cache contentTypeCache + for i, record := range td.Contents { + var ( + c Content + err error + ) + if dec != nil { + if c, err = dec(record.Payload); err != nil { + return nil, fmt.Errorf("merkletree: unmarshaling content at index %d: %w", i, err) + } + } else if c, err = unmarshalRegisteredContent(record.Type, record.Payload, &cache); err != nil { + return nil, err + } + if c == nil { + return nil, fmt.Errorf("merkletree: content at index %d decoded to nil", i) + } + cs = append(cs, c) + } + + if td.Sort && td.RFC6962 { + return nil, fmt.Errorf("%w: both the sort and RFC 6962 flags are set, which no tree can be built with", ErrCorruptData) + } + + t := &MerkleTree{ + hashStrategy: strategy, + hashStrategyName: td.HashStrategy, + sort: td.Sort, + rfc6962: td.RFC6962, + } + root, leafs, err := buildWithContent(cs, t) + if err != nil { + return nil, err + } + // The recorded root makes decoding self-checking. Content that decoded to + // something other than what was encoded, a hash strategy that does not match the + // one the payload was written with, and bit rot anywhere in the payload all land + // here rather than producing a tree that looks fine and verifies against nothing. + if !bytes.Equal(root.Hash, td.MerkleRoot) { + return nil, fmt.Errorf("%w: rebuilt %x, encoded %x", ErrRootMismatch, root.Hash, td.MerkleRoot) + } + + t.Root = root + t.Leafs = leafs + t.merkleRoot = root.Hash + return t, nil +} + +// MarshalWith encodes the tree, using enc to encode each content item. It requires no +// package-level registration, which makes it the right choice for libraries and for +// content types that already have an encoding of their own. +// +// Only the leaf content, the hash strategy name, the sibling sort flag, and the Merkle +// root are written; the tree structure is rebuilt on decode. Pair this with +// UnmarshalWith and a decoder that is the exact inverse of enc. +func (m MerkleTree) MarshalWith(enc ContentMarshalFunc, opts ...MarshalOption) ([]byte, error) { + if enc == nil { + return nil, errors.New("merkletree: MarshalWith requires a content marshal function") + } + td, err := m.snapshot(enc, opts...) + if err != nil { + return nil, err + } + return td.marshalBinary(), nil +} + +// UnmarshalWith decodes a tree written by MarshalWith, using dec to decode each content +// item. The rebuilt tree is verified against the Merkle root recorded in data, so a +// successful return means the tree hashes exactly as it did when it was encoded. +func UnmarshalWith(data []byte, dec ContentUnmarshalFunc, opts ...UnmarshalOption) (*MerkleTree, error) { + if dec == nil { + return nil, errors.New("merkletree: UnmarshalWith requires a content unmarshal function") + } + td, err := unmarshalTreeData(data) + if err != nil { + return nil, err + } + return td.tree(dec, opts...) +} + +// MarshalBinary encodes the tree using the package content registry, implementing +// encoding.BinaryMarshaler. This is what makes a tree work with encoding/gob and any +// other codec that honors BinaryMarshaler: +// +// merkletree.RegisterContent(MyContent{}) +// err := gob.NewEncoder(&buf).Encode(tree) +// +// Every content type in the tree must have been registered with RegisterContent, and +// the tree's hash strategy must have been registered with RegisterHashStrategy. Use +// MarshalWith to avoid the registries. +func (m MerkleTree) MarshalBinary() ([]byte, error) { + td, err := m.snapshot(nil) + if err != nil { + return nil, err + } + return td.marshalBinary(), nil +} + +// UnmarshalBinary rebuilds the tree from a payload written by MarshalBinary, +// implementing encoding.BinaryUnmarshaler. The receiver is left untouched if decoding +// fails for any reason, including the rebuilt root failing to match the recorded one. +func (m *MerkleTree) UnmarshalBinary(data []byte) error { + td, err := unmarshalTreeData(data) + if err != nil { + return err + } + t, err := td.tree(nil) + if err != nil { + return err + } + *m = *t + m.adoptNodes() + return nil +} + +// MarshalJSON encodes the tree using the package content registry, implementing +// json.Marshaler. Byte fields are base64 encoded by encoding/json as usual. +// +// Without this method json.Marshal would recurse forever on the tree's parent and tree +// back-pointers. Content is encoded through encoding.BinaryMarshaler, the same as +// MarshalBinary, so the JSON form carries opaque content payloads rather than the +// content's own JSON shape. Use MarshalWith when you need control over that. +func (m MerkleTree) MarshalJSON() ([]byte, error) { + td, err := m.snapshot(nil) + if err != nil { + return nil, err + } + return json.Marshal(td) +} + +// UnmarshalJSON rebuilds the tree from a payload written by MarshalJSON, implementing +// json.Unmarshaler. As with UnmarshalBinary the receiver is left untouched on failure. +func (m *MerkleTree) UnmarshalJSON(data []byte) error { + var td treeData + if err := json.Unmarshal(data, &td); err != nil { + return fmt.Errorf("%w: %w", ErrCorruptData, err) + } + t, err := td.tree(nil) + if err != nil { + return err + } + *m = *t + m.adoptNodes() + return nil +} + +// adoptNodes re-points every node's Tree back-pointer at m. The unmarshalers build into +// a scratch tree so that a failure leaves the receiver alone, which leaves the nodes +// referring to that scratch tree once its fields are copied over. +// +// Walking up from the leaves reaches every node, and stopping as soon as an ancestor +// has already been adopted keeps the whole pass linear. +func (m *MerkleTree) adoptNodes() { + for _, l := range m.Leafs { + for n := l; n != nil && n.Tree != m; n = n.Parent { + n.Tree = m + } + } +} + +// marshalBinary writes the seed in a compact, deterministic, self-describing format: +// +// magic "MTREE" +// version uvarint +// strategy uvarint length + bytes +// sort one byte, 0 or 1 +// rfc6962 one byte, 0 or 1 +// merkleRoot uvarint length + bytes +// count uvarint +// type uvarint length + bytes (repeated count times) +// payload uvarint length + bytes +// +// Encoding the same tree twice always produces identical bytes, so payloads can be +// compared or content-addressed directly. +func (td *treeData) marshalBinary() []byte { + // The wire size is exactly computable before writing a byte, so the buffer is + // grown to it once. Without this the buffer doubles its way up, which on a large + // tree re-copies the payload log n times and leaves the final allocation up to + // twice the size it needs to be. + size := len(serializationMagic) + + uvarintLen(uint64(td.Version)) + + uvarintLen(uint64(len(td.HashStrategy))) + len(td.HashStrategy) + + 2 + + uvarintLen(uint64(len(td.MerkleRoot))) + len(td.MerkleRoot) + + uvarintLen(uint64(len(td.Contents))) + for _, record := range td.Contents { + size += uvarintLen(uint64(len(record.Type))) + len(record.Type) + + uvarintLen(uint64(len(record.Payload))) + len(record.Payload) + } + + var buf bytes.Buffer + buf.Grow(size) + buf.WriteString(serializationMagic) + writeUvarint(&buf, uint64(td.Version)) + writeBytes(&buf, []byte(td.HashStrategy)) + if td.Sort { + buf.WriteByte(1) + } else { + buf.WriteByte(0) + } + if td.RFC6962 { + buf.WriteByte(1) + } else { + buf.WriteByte(0) + } + writeBytes(&buf, td.MerkleRoot) + writeUvarint(&buf, uint64(len(td.Contents))) + for _, record := range td.Contents { + writeBytes(&buf, []byte(record.Type)) + writeBytes(&buf, record.Payload) + } + return buf.Bytes() +} + +// unmarshalTreeData parses the format written by marshalBinary. Every length is checked +// against the bytes actually remaining before it is used to allocate, so a corrupt or +// hostile payload fails rather than exhausting memory. +func unmarshalTreeData(data []byte) (*treeData, error) { + if len(data) < len(serializationMagic) || string(data[:len(serializationMagic)]) != serializationMagic { + return nil, fmt.Errorf("%w: missing %q header", ErrCorruptData, serializationMagic) + } + r := &binaryReader{data: data[len(serializationMagic):]} + + version, err := r.uvarint() + if err != nil { + return nil, fmt.Errorf("%w: reading version: %w", ErrCorruptData, err) + } + if version != serializationVersion { + return nil, fmt.Errorf("%w: got %d, this build writes and reads %d", ErrUnsupportedVersion, version, serializationVersion) + } + + td := &treeData{Version: int(version)} + + strategy, err := r.view() + if err != nil { + return nil, fmt.Errorf("%w: reading hash strategy: %w", ErrCorruptData, err) + } + td.HashStrategy = string(strategy) + + sortFlag, err := r.readByte() + if err != nil { + return nil, fmt.Errorf("%w: reading sort flag: %w", ErrCorruptData, err) + } + if sortFlag > 1 { + return nil, fmt.Errorf("%w: sort flag is %d, expected 0 or 1", ErrCorruptData, sortFlag) + } + td.Sort = sortFlag == 1 + + rfcFlag, err := r.readByte() + if err != nil { + return nil, fmt.Errorf("%w: reading RFC 6962 flag: %w", ErrCorruptData, err) + } + if rfcFlag > 1 { + return nil, fmt.Errorf("%w: RFC 6962 flag is %d, expected 0 or 1", ErrCorruptData, rfcFlag) + } + td.RFC6962 = rfcFlag == 1 + + // The root is compared once while the tree is rebuilt and then dropped with the + // treeData carrying it, so viewing it in place is safe; nothing retains it. + if td.MerkleRoot, err = r.view(); err != nil { + return nil, fmt.Errorf("%w: reading Merkle root: %w", ErrCorruptData, err) + } + + count, err := r.uvarint() + if err != nil { + return nil, fmt.Errorf("%w: reading content count: %w", ErrCorruptData, err) + } + // Each record costs at least two length bytes, so a count larger than the bytes + // remaining cannot be honest. This bounds the allocations below. + if count > uint64(r.remaining()) { + return nil, fmt.Errorf("%w: content count %d exceeds the %d bytes remaining", ErrCorruptData, count, r.remaining()) + } + + td.Contents = make([]contentRecord, 0, count) + // Every record carries its type name, and in practice a tree holds one content + // type, so the same name is repeated for every leaf. Interning them means one + // string for the whole payload rather than one per record. Indexing a map with + // string(byteSlice) does not allocate, so a repeat name costs only the lookup. + interned := make(map[string]string, 1) + // One arena serves every payload. UnmarshalBinary implementations are permitted + // to retain the slice they are handed, so a payload must not alias the caller's + // data - but copying each into its own allocation cost one per record. Carving + // copies out of a single arena keeps the no-aliasing guarantee at one allocation + // for the whole decode. Each carve is capped at its own length, so appending to + // one payload cannot reach the record stored after it, and the contents all live + // and die with the tree they decode into, so the shared backing pins nothing + // that was not already pinned. The remaining bytes bound the payload bytes, so + // the arena never grows and every carve stays inside the one allocation. + arena := make([]byte, 0, r.remaining()) + for i := uint64(0); i < count; i++ { + typeName, err := r.view() + if err != nil { + return nil, fmt.Errorf("%w: reading content type at index %d: %w", ErrCorruptData, i, err) + } + name, ok := interned[string(typeName)] + if !ok { + name = string(typeName) + interned[name] = name + } + view, err := r.view() + if err != nil { + return nil, fmt.Errorf("%w: reading content payload at index %d: %w", ErrCorruptData, i, err) + } + off := len(arena) + arena = append(arena, view...) + td.Contents = append(td.Contents, contentRecord{Type: name, Payload: arena[off:len(arena):len(arena)]}) + } + + if r.remaining() != 0 { + return nil, fmt.Errorf("%w: %d trailing bytes after the last content record", ErrCorruptData, r.remaining()) + } + return td, nil +} + +// binaryReader reads the length-prefixed pieces of the wire format. It is a cursor +// over the payload slice rather than a bytes.Reader: every read is a bounds check and +// a reslice with no interface calls, and a field that is only inspected rather than +// kept - a type name about to be interned, the recorded root - can be viewed in place +// instead of copied out. +type binaryReader struct { + data []byte + off int +} + +// remaining reports how many bytes are left to read. +func (br *binaryReader) remaining() int { + return len(br.data) - br.off +} + +func (br *binaryReader) readByte() (byte, error) { + if br.off >= len(br.data) { + return 0, io.EOF + } + b := br.data[br.off] + br.off++ + return b, nil +} + +// view returns the next length-prefixed field as a subslice of the payload, without +// copying. The result aliases the caller's input; anything that outlives the decode +// must copy what it keeps. +func (br *binaryReader) view() ([]byte, error) { + n, err := br.uvarint() + if err != nil { + return nil, err + } + if n > uint64(br.remaining()) { + return nil, fmt.Errorf("length %d exceeds the %d bytes remaining", n, br.remaining()) + } + b := br.data[br.off : br.off+int(n) : br.off+int(n)] + br.off += int(n) + return b, nil +} + +// uvarint reads one uvarint, rejecting any encoding that is not the shortest one for +// the value it carries. +// +// binary.ReadUvarint accepts padded forms - 0x80 0x00 decodes to zero just as 0x00 +// does - which would make the wire format malleable: a payload could be rewritten +// into different bytes that decode to exactly the same tree. Since marshalBinary is +// documented as deterministic so that payloads can be compared or content addressed, +// the decoder has to hold up the other half of that guarantee. +func (br *binaryReader) uvarint() (uint64, error) { + var x uint64 + var s uint + for i := 0; ; i++ { + b, err := br.readByte() + if err != nil { + if i > 0 && errors.Is(err, io.EOF) { + err = io.ErrUnexpectedEOF + } + return 0, err + } + if i == binary.MaxVarintLen64-1 && b > 1 { + return 0, errors.New("uvarint overflows a 64 bit value") + } + if b < 0x80 { + // The final byte holds the most significant group. A zero group means + // the same value had a shorter encoding, so this one is not canonical. + if i > 0 && b == 0 { + return 0, errors.New("uvarint is not minimally encoded") + } + + return x | uint64(b)<= 0x80 { + v >>= 7 + n++ + } + return n +} + +func writeUvarint(buf *bytes.Buffer, v uint64) { + var scratch [binary.MaxVarintLen64]byte + buf.Write(scratch[:binary.PutUvarint(scratch[:], v)]) +} + +func writeBytes(buf *bytes.Buffer, b []byte) { + writeUvarint(buf, uint64(len(b))) + buf.Write(b) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index a049ce69..470ca25e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -181,8 +181,8 @@ github.com/aws/smithy-go/waiter # github.com/beorn7/perks v1.0.1 ## explicit; go 1.11 github.com/beorn7/perks/quantile -# github.com/cbergoon/merkletree v0.2.0 -## explicit; go 1.12 +# github.com/cbergoon/merkletree v0.5.0 +## explicit; go 1.21 github.com/cbergoon/merkletree # github.com/cenkalti/backoff/v4 v4.3.0 ## explicit; go 1.18