From fd296811df7cf036c6c43665e92896df5db7c573 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 21 Apr 2017 15:33:25 -0400 Subject: [PATCH] simple merkle tree --- merkle/README.md | 4 + merkle/simple_tree.go | 277 +++++++++++++++++++++++++++++++++++++ merkle/simple_tree_test.go | 87 ++++++++++++ merkle/types.go | 23 +++ 4 files changed, 391 insertions(+) create mode 100644 merkle/README.md create mode 100644 merkle/simple_tree.go create mode 100644 merkle/simple_tree_test.go create mode 100644 merkle/types.go diff --git a/merkle/README.md b/merkle/README.md new file mode 100644 index 00000000..c4497836 --- /dev/null +++ b/merkle/README.md @@ -0,0 +1,4 @@ +## Simple Merkle Tree + +For smaller static data structures that don't require immutable snapshots or mutability; +for instance the transactions and validation signatures of a block can be hashed using this simple merkle tree logic. diff --git a/merkle/simple_tree.go b/merkle/simple_tree.go new file mode 100644 index 00000000..51733144 --- /dev/null +++ b/merkle/simple_tree.go @@ -0,0 +1,277 @@ +/* +Computes a deterministic minimal height merkle tree hash. +If the number of items is not a power of two, some leaves +will be at different levels. Tries to keep both sides of +the tree the same size, but the left may be one greater. + +Use this for short deterministic trees, such as the validator list. +For larger datasets, use IAVLTree. + + * + / \ + / \ + / \ + / \ + * * + / \ / \ + / \ / \ + / \ / \ + * * * h6 + / \ / \ / \ + h0 h1 h2 h3 h4 h5 + +*/ + +package merkle + +import ( + "bytes" + "fmt" + "sort" + + "golang.org/x/crypto/ripemd160" + + . "github.com/tendermint/go-common" + "github.com/tendermint/go-wire" +) + +func SimpleHashFromTwoHashes(left []byte, right []byte) []byte { + var n int + var err error + var hasher = ripemd160.New() + wire.WriteByteSlice(left, hasher, &n, &err) + wire.WriteByteSlice(right, hasher, &n, &err) + if err != nil { + PanicCrisis(err) + } + return hasher.Sum(nil) +} + +func SimpleHashFromHashes(hashes [][]byte) []byte { + // Recursive impl. + switch len(hashes) { + case 0: + return nil + case 1: + return hashes[0] + default: + left := SimpleHashFromHashes(hashes[:(len(hashes)+1)/2]) + right := SimpleHashFromHashes(hashes[(len(hashes)+1)/2:]) + return SimpleHashFromTwoHashes(left, right) + } +} + +// Convenience for SimpleHashFromHashes. +func SimpleHashFromBinaries(items []interface{}) []byte { + hashes := make([][]byte, len(items)) + for i, item := range items { + hashes[i] = SimpleHashFromBinary(item) + } + return SimpleHashFromHashes(hashes) +} + +// General Convenience +func SimpleHashFromBinary(item interface{}) []byte { + hasher, n, err := ripemd160.New(), new(int), new(error) + wire.WriteBinary(item, hasher, n, err) + if *err != nil { + PanicCrisis(err) + } + return hasher.Sum(nil) +} + +// Convenience for SimpleHashFromHashes. +func SimpleHashFromHashables(items []Hashable) []byte { + hashes := make([][]byte, len(items)) + for i, item := range items { + hash := item.Hash() + hashes[i] = hash + } + return SimpleHashFromHashes(hashes) +} + +// Convenience for SimpleHashFromHashes. +func SimpleHashFromMap(m map[string]interface{}) []byte { + kpPairsH := MakeSortedKVPairs(m) + return SimpleHashFromHashables(kpPairsH) +} + +//-------------------------------------------------------------------------------- + +/* Convenience struct for key-value pairs. +A list of KVPairs is hashed via `SimpleHashFromHashables`. +NOTE: Each `Value` is encoded for hashing without extra type information, +so the user is presumed to be aware of the Value types. +*/ +type KVPair struct { + Key string + Value interface{} +} + +func (kv KVPair) Hash() []byte { + hasher, n, err := ripemd160.New(), new(int), new(error) + wire.WriteString(kv.Key, hasher, n, err) + if kvH, ok := kv.Value.(Hashable); ok { + wire.WriteByteSlice(kvH.Hash(), hasher, n, err) + } else { + wire.WriteBinary(kv.Value, hasher, n, err) + } + if *err != nil { + PanicSanity(*err) + } + return hasher.Sum(nil) +} + +type KVPairs []KVPair + +func (kvps KVPairs) Len() int { return len(kvps) } +func (kvps KVPairs) Less(i, j int) bool { return kvps[i].Key < kvps[j].Key } +func (kvps KVPairs) Swap(i, j int) { kvps[i], kvps[j] = kvps[j], kvps[i] } +func (kvps KVPairs) Sort() { sort.Sort(kvps) } + +func MakeSortedKVPairs(m map[string]interface{}) []Hashable { + kvPairs := []KVPair{} + for k, v := range m { + kvPairs = append(kvPairs, KVPair{k, v}) + } + KVPairs(kvPairs).Sort() + kvPairsH := []Hashable{} + for _, kvp := range kvPairs { + kvPairsH = append(kvPairsH, kvp) + } + return kvPairsH +} + +//-------------------------------------------------------------------------------- + +type SimpleProof struct { + Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. +} + +// proofs[0] is the proof for items[0]. +func SimpleProofsFromHashables(items []Hashable) (rootHash []byte, proofs []*SimpleProof) { + trails, rootSPN := trailsFromHashables(items) + rootHash = rootSPN.Hash + proofs = make([]*SimpleProof, len(items)) + for i, trail := range trails { + proofs[i] = &SimpleProof{ + Aunts: trail.FlattenAunts(), + } + } + return +} + +// Verify that leafHash is a leaf hash of the simple-merkle-tree +// which hashes to rootHash. +func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash []byte) bool { + computedHash := computeHashFromAunts(index, total, leafHash, sp.Aunts) + if computedHash == nil { + return false + } + if !bytes.Equal(computedHash, rootHash) { + return false + } + return true +} + +func (sp *SimpleProof) String() string { + return sp.StringIndented("") +} + +func (sp *SimpleProof) StringIndented(indent string) string { + return fmt.Sprintf(`SimpleProof{ +%s Aunts: %X +%s}`, + indent, sp.Aunts, + indent) +} + +// Use the leafHash and innerHashes to get the root merkle hash. +// If the length of the innerHashes slice isn't exactly correct, the result is nil. +func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][]byte) []byte { + // Recursive impl. + if index >= total { + return nil + } + switch total { + case 0: + PanicSanity("Cannot call computeHashFromAunts() with 0 total") + return nil + case 1: + if len(innerHashes) != 0 { + return nil + } + return leafHash + default: + if len(innerHashes) == 0 { + return nil + } + numLeft := (total + 1) / 2 + if index < numLeft { + leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if leftHash == nil { + return nil + } + return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + } else { + rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if rightHash == nil { + return nil + } + return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) + } + } +} + +// Helper structure to construct merkle proof. +// The node and the tree is thrown away afterwards. +// Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. +// node.Parent.Hash = hash(node.Hash, node.Right.Hash) or +// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. +type SimpleProofNode struct { + Hash []byte + Parent *SimpleProofNode + Left *SimpleProofNode // Left sibling (only one of Left,Right is set) + Right *SimpleProofNode // Right sibling (only one of Left,Right is set) +} + +// Starting from a leaf SimpleProofNode, FlattenAunts() will return +// the inner hashes for the item corresponding to the leaf. +func (spn *SimpleProofNode) FlattenAunts() [][]byte { + // Nonrecursive impl. + innerHashes := [][]byte{} + for spn != nil { + if spn.Left != nil { + innerHashes = append(innerHashes, spn.Left.Hash) + } else if spn.Right != nil { + innerHashes = append(innerHashes, spn.Right.Hash) + } else { + break + } + spn = spn.Parent + } + return innerHashes +} + +// trails[0].Hash is the leaf hash for items[0]. +// trails[i].Parent.Parent....Parent == root for all i. +func trailsFromHashables(items []Hashable) (trails []*SimpleProofNode, root *SimpleProofNode) { + // Recursive impl. + switch len(items) { + case 0: + return nil, nil + case 1: + trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil} + return []*SimpleProofNode{trail}, trail + default: + lefts, leftRoot := trailsFromHashables(items[:(len(items)+1)/2]) + rights, rightRoot := trailsFromHashables(items[(len(items)+1)/2:]) + rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) + root := &SimpleProofNode{rootHash, nil, nil, nil} + leftRoot.Parent = root + leftRoot.Right = rightRoot + rightRoot.Parent = root + rightRoot.Left = leftRoot + return append(lefts, rights...), root + } +} diff --git a/merkle/simple_tree_test.go b/merkle/simple_tree_test.go new file mode 100644 index 00000000..424b498f --- /dev/null +++ b/merkle/simple_tree_test.go @@ -0,0 +1,87 @@ +package merkle + +import ( + "bytes" + + . "github.com/tendermint/go-common" + . "github.com/tendermint/go-common/test" + + "testing" +) + +type testItem []byte + +func (tI testItem) Hash() []byte { + return []byte(tI) +} + +func TestSimpleProof(t *testing.T) { + + total := 100 + + items := make([]Hashable, total) + for i := 0; i < total; i++ { + items[i] = testItem(RandBytes(32)) + } + + rootHash := SimpleHashFromHashables(items) + + rootHash2, proofs := SimpleProofsFromHashables(items) + + if !bytes.Equal(rootHash, rootHash2) { + t.Errorf("Unmatched root hashes: %X vs %X", rootHash, rootHash2) + } + + // For each item, check the trail. + for i, item := range items { + itemHash := item.Hash() + proof := proofs[i] + + // Verify success + ok := proof.Verify(i, total, itemHash, rootHash) + if !ok { + t.Errorf("Verification failed for index %v.", i) + } + + // Wrong item index should make it fail + { + ok = proof.Verify((i+1)%total, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong index %v.", i) + } + } + + // Trail too long should make it fail + origAunts := proof.Aunts + proof.Aunts = append(proof.Aunts, RandBytes(32)) + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Trail too short should make it fail + proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1] + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Mutating the itemHash should make it fail. + ok = proof.Verify(i, total, MutateByteSlice(itemHash), rootHash) + if ok { + t.Errorf("Expected verification to fail for mutated leaf hash") + } + + // Mutating the rootHash should make it fail. + ok = proof.Verify(i, total, itemHash, MutateByteSlice(rootHash)) + if ok { + t.Errorf("Expected verification to fail for mutated root hash") + } + } +} diff --git a/merkle/types.go b/merkle/types.go new file mode 100644 index 00000000..93541eda --- /dev/null +++ b/merkle/types.go @@ -0,0 +1,23 @@ +package merkle + +type Tree interface { + Size() (size int) + Height() (height int8) + Has(key []byte) (has bool) + Proof(key []byte) (value []byte, proof []byte, exists bool) // TODO make it return an index + Get(key []byte) (index int, value []byte, exists bool) + GetByIndex(index int) (key []byte, value []byte) + Set(key []byte, value []byte) (updated bool) + Remove(key []byte) (value []byte, removed bool) + HashWithCount() (hash []byte, count int) + Hash() (hash []byte) + Save() (hash []byte) + Load(hash []byte) + Copy() Tree + Iterate(func(key []byte, value []byte) (stop bool)) (stopped bool) + IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool) +} + +type Hashable interface { + Hash() []byte +}