From 134fdf716930c308ac9f6cd3a80f55ee57f40254 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 22 May 2018 12:43:44 -0400 Subject: [PATCH 1/6] bring in merkle from tmlibs --- merkle/README.md | 4 ++ merkle/simple_map.go | 84 ++++++++++++++++++++++ merkle/simple_map_test.go | 53 ++++++++++++++ merkle/simple_proof.go | 144 +++++++++++++++++++++++++++++++++++++ merkle/simple_tree.go | 91 +++++++++++++++++++++++ merkle/simple_tree_test.go | 87 ++++++++++++++++++++++ merkle/types.go | 47 ++++++++++++ 7 files changed, 510 insertions(+) create mode 100644 merkle/README.md create mode 100644 merkle/simple_map.go create mode 100644 merkle/simple_map_test.go create mode 100644 merkle/simple_proof.go create mode 100644 merkle/simple_tree.go create mode 100644 merkle/simple_tree_test.go create mode 100644 merkle/types.go diff --git a/merkle/README.md b/merkle/README.md new file mode 100644 index 00000000..c4497836 --- /dev/null +++ b/merkle/README.md @@ -0,0 +1,4 @@ +## Simple Merkle Tree + +For smaller static data structures that don't require immutable snapshots or mutability; +for instance the transactions and validation signatures of a block can be hashed using this simple merkle tree logic. diff --git a/merkle/simple_map.go b/merkle/simple_map.go new file mode 100644 index 00000000..cd38de76 --- /dev/null +++ b/merkle/simple_map.go @@ -0,0 +1,84 @@ +package merkle + +import ( + cmn "github.com/tendermint/tmlibs/common" + "golang.org/x/crypto/ripemd160" +) + +type SimpleMap struct { + kvs cmn.KVPairs + sorted bool +} + +func NewSimpleMap() *SimpleMap { + return &SimpleMap{ + kvs: nil, + sorted: false, + } +} + +func (sm *SimpleMap) Set(key string, value Hasher) { + sm.sorted = false + + // Hash the key to blind it... why not? + khash := SimpleHashFromBytes([]byte(key)) + + // And the value is hashed too, so you can + // check for equality with a cached value (say) + // and make a determination to fetch or not. + vhash := value.Hash() + + sm.kvs = append(sm.kvs, cmn.KVPair{ + Key: khash, + Value: vhash, + }) +} + +// Merkle root hash of items sorted by key +// (UNSTABLE: and by value too if duplicate key). +func (sm *SimpleMap) Hash() []byte { + sm.Sort() + return hashKVPairs(sm.kvs) +} + +func (sm *SimpleMap) Sort() { + if sm.sorted { + return + } + sm.kvs.Sort() + sm.sorted = true +} + +// Returns a copy of sorted KVPairs. +func (sm *SimpleMap) KVPairs() cmn.KVPairs { + sm.Sort() + kvs := make(cmn.KVPairs, len(sm.kvs)) + copy(kvs, sm.kvs) + return kvs +} + +//---------------------------------------- + +// A local extension to KVPair that can be hashed. +type KVPair cmn.KVPair + +func (kv KVPair) Hash() []byte { + hasher := ripemd160.New() + err := encodeByteSlice(hasher, kv.Key) + if err != nil { + panic(err) + } + err = encodeByteSlice(hasher, kv.Value) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + +func hashKVPairs(kvs cmn.KVPairs) []byte { + kvsH := make([]Hasher, 0, len(kvs)) + for _, kvp := range kvs { + kvsH = append(kvsH, KVPair(kvp)) + } + return SimpleHashFromHashers(kvsH) +} diff --git a/merkle/simple_map_test.go b/merkle/simple_map_test.go new file mode 100644 index 00000000..c9c87135 --- /dev/null +++ b/merkle/simple_map_test.go @@ -0,0 +1,53 @@ +package merkle + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +type strHasher string + +func (str strHasher) Hash() []byte { + return SimpleHashFromBytes([]byte(str)) +} + +func TestSimpleMap(t *testing.T) { + { + db := NewSimpleMap() + db.Set("key1", strHasher("value1")) + assert.Equal(t, "acdb4f121bc6f25041eb263ab463f1cd79236a32", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := NewSimpleMap() + db.Set("key1", strHasher("value2")) + assert.Equal(t, "b8cbf5adee8c524e14f531da9b49adbbbd66fffa", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := NewSimpleMap() + db.Set("key1", strHasher("value1")) + db.Set("key2", strHasher("value2")) + assert.Equal(t, "1708aabc85bbe00242d3db8c299516aa54e48c38", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := NewSimpleMap() + db.Set("key2", strHasher("value2")) // NOTE: out of order + db.Set("key1", strHasher("value1")) + assert.Equal(t, "1708aabc85bbe00242d3db8c299516aa54e48c38", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := NewSimpleMap() + db.Set("key1", strHasher("value1")) + db.Set("key2", strHasher("value2")) + db.Set("key3", strHasher("value3")) + assert.Equal(t, "e728afe72ce351eed6aca65c5f78da19b9a6e214", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } + { + db := NewSimpleMap() + db.Set("key2", strHasher("value2")) // NOTE: out of order + db.Set("key1", strHasher("value1")) + db.Set("key3", strHasher("value3")) + assert.Equal(t, "e728afe72ce351eed6aca65c5f78da19b9a6e214", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + } +} diff --git a/merkle/simple_proof.go b/merkle/simple_proof.go new file mode 100644 index 00000000..ca6ccf37 --- /dev/null +++ b/merkle/simple_proof.go @@ -0,0 +1,144 @@ +package merkle + +import ( + "bytes" + "fmt" +) + +type SimpleProof struct { + Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. +} + +// proofs[0] is the proof for items[0]. +func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) { + trails, rootSPN := trailsFromHashers(items) + rootHash = rootSPN.Hash + proofs = make([]*SimpleProof, len(items)) + for i, trail := range trails { + proofs[i] = &SimpleProof{ + Aunts: trail.FlattenAunts(), + } + } + return +} + +func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs []*SimpleProof) { + sm := NewSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + sm.Sort() + kvs := sm.kvs + kvsH := make([]Hasher, 0, len(kvs)) + for _, kvp := range kvs { + kvsH = append(kvsH, KVPair(kvp)) + } + return SimpleProofsFromHashers(kvsH) +} + +// Verify that leafHash is a leaf hash of the simple-merkle-tree +// which hashes to rootHash. +func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash []byte) bool { + computedHash := computeHashFromAunts(index, total, leafHash, sp.Aunts) + return computedHash != nil && bytes.Equal(computedHash, rootHash) +} + +func (sp *SimpleProof) String() string { + return sp.StringIndented("") +} + +func (sp *SimpleProof) StringIndented(indent string) string { + return fmt.Sprintf(`SimpleProof{ +%s Aunts: %X +%s}`, + indent, sp.Aunts, + indent) +} + +// Use the leafHash and innerHashes to get the root merkle hash. +// If the length of the innerHashes slice isn't exactly correct, the result is nil. +// Recursive impl. +func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][]byte) []byte { + if index >= total || index < 0 || total <= 0 { + return nil + } + switch total { + case 0: + panic("Cannot call computeHashFromAunts() with 0 total") + case 1: + if len(innerHashes) != 0 { + return nil + } + return leafHash + default: + if len(innerHashes) == 0 { + return nil + } + numLeft := (total + 1) / 2 + if index < numLeft { + leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if leftHash == nil { + return nil + } + return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + } + rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) + if rightHash == nil { + return nil + } + return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) + } +} + +// Helper structure to construct merkle proof. +// The node and the tree is thrown away afterwards. +// Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. +// node.Parent.Hash = hash(node.Hash, node.Right.Hash) or +// hash(node.Left.Hash, node.Hash), depending on whether node is a left/right child. +type SimpleProofNode struct { + Hash []byte + Parent *SimpleProofNode + Left *SimpleProofNode // Left sibling (only one of Left,Right is set) + Right *SimpleProofNode // Right sibling (only one of Left,Right is set) +} + +// Starting from a leaf SimpleProofNode, FlattenAunts() will return +// the inner hashes for the item corresponding to the leaf. +func (spn *SimpleProofNode) FlattenAunts() [][]byte { + // Nonrecursive impl. + innerHashes := [][]byte{} + for spn != nil { + if spn.Left != nil { + innerHashes = append(innerHashes, spn.Left.Hash) + } else if spn.Right != nil { + innerHashes = append(innerHashes, spn.Right.Hash) + } else { + break + } + spn = spn.Parent + } + return innerHashes +} + +// trails[0].Hash is the leaf hash for items[0]. +// trails[i].Parent.Parent....Parent == root for all i. +func trailsFromHashers(items []Hasher) (trails []*SimpleProofNode, root *SimpleProofNode) { + // Recursive impl. + switch len(items) { + case 0: + return nil, nil + case 1: + trail := &SimpleProofNode{items[0].Hash(), nil, nil, nil} + return []*SimpleProofNode{trail}, trail + default: + lefts, leftRoot := trailsFromHashers(items[:(len(items)+1)/2]) + rights, rightRoot := trailsFromHashers(items[(len(items)+1)/2:]) + rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) + root := &SimpleProofNode{rootHash, nil, nil, nil} + leftRoot.Parent = root + leftRoot.Right = rightRoot + rightRoot.Parent = root + rightRoot.Left = leftRoot + return append(lefts, rights...), root + } +} diff --git a/merkle/simple_tree.go b/merkle/simple_tree.go new file mode 100644 index 00000000..9bdf52cb --- /dev/null +++ b/merkle/simple_tree.go @@ -0,0 +1,91 @@ +/* +Computes a deterministic minimal height merkle tree hash. +If the number of items is not a power of two, some leaves +will be at different levels. Tries to keep both sides of +the tree the same size, but the left may be one greater. + +Use this for short deterministic trees, such as the validator list. +For larger datasets, use IAVLTree. + + * + / \ + / \ + / \ + / \ + * * + / \ / \ + / \ / \ + / \ / \ + * * * h6 + / \ / \ / \ + h0 h1 h2 h3 h4 h5 + +*/ + +package merkle + +import ( + "golang.org/x/crypto/ripemd160" +) + +func SimpleHashFromTwoHashes(left []byte, right []byte) []byte { + var hasher = ripemd160.New() + err := encodeByteSlice(hasher, left) + if err != nil { + panic(err) + } + err = encodeByteSlice(hasher, right) + if err != nil { + panic(err) + } + return hasher.Sum(nil) +} + +func SimpleHashFromHashes(hashes [][]byte) []byte { + // Recursive impl. + switch len(hashes) { + case 0: + return nil + case 1: + return hashes[0] + default: + left := SimpleHashFromHashes(hashes[:(len(hashes)+1)/2]) + right := SimpleHashFromHashes(hashes[(len(hashes)+1)/2:]) + return SimpleHashFromTwoHashes(left, right) + } +} + +// NOTE: Do not implement this, use SimpleHashFromByteslices instead. +// type Byteser interface { Bytes() []byte } +// func SimpleHashFromBytesers(items []Byteser) []byte { ... } + +func SimpleHashFromByteslices(bzs [][]byte) []byte { + hashes := make([][]byte, len(bzs)) + for i, bz := range bzs { + hashes[i] = SimpleHashFromBytes(bz) + } + return SimpleHashFromHashes(hashes) +} + +func SimpleHashFromBytes(bz []byte) []byte { + hasher := ripemd160.New() + hasher.Write(bz) + return hasher.Sum(nil) +} + +func SimpleHashFromHashers(items []Hasher) []byte { + hashes := make([][]byte, len(items)) + for i, item := range items { + hash := item.Hash() + hashes[i] = hash + } + return SimpleHashFromHashes(hashes) +} + +func SimpleHashFromMap(m map[string]Hasher) []byte { + sm := NewSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + return sm.Hash() +} diff --git a/merkle/simple_tree_test.go b/merkle/simple_tree_test.go new file mode 100644 index 00000000..8c4ed01f --- /dev/null +++ b/merkle/simple_tree_test.go @@ -0,0 +1,87 @@ +package merkle + +import ( + "bytes" + + cmn "github.com/tendermint/tmlibs/common" + . "github.com/tendermint/tmlibs/test" + + "testing" +) + +type testItem []byte + +func (tI testItem) Hash() []byte { + return []byte(tI) +} + +func TestSimpleProof(t *testing.T) { + + total := 100 + + items := make([]Hasher, total) + for i := 0; i < total; i++ { + items[i] = testItem(cmn.RandBytes(32)) + } + + rootHash := SimpleHashFromHashers(items) + + rootHash2, proofs := SimpleProofsFromHashers(items) + + if !bytes.Equal(rootHash, rootHash2) { + t.Errorf("Unmatched root hashes: %X vs %X", rootHash, rootHash2) + } + + // For each item, check the trail. + for i, item := range items { + itemHash := item.Hash() + proof := proofs[i] + + // Verify success + ok := proof.Verify(i, total, itemHash, rootHash) + if !ok { + t.Errorf("Verification failed for index %v.", i) + } + + // Wrong item index should make it fail + { + ok = proof.Verify((i+1)%total, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong index %v.", i) + } + } + + // Trail too long should make it fail + origAunts := proof.Aunts + proof.Aunts = append(proof.Aunts, cmn.RandBytes(32)) + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Trail too short should make it fail + proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1] + { + ok = proof.Verify(i, total, itemHash, rootHash) + if ok { + t.Errorf("Expected verification to fail for wrong trail length.") + } + } + proof.Aunts = origAunts + + // Mutating the itemHash should make it fail. + ok = proof.Verify(i, total, MutateByteSlice(itemHash), rootHash) + if ok { + t.Errorf("Expected verification to fail for mutated leaf hash") + } + + // Mutating the rootHash should make it fail. + ok = proof.Verify(i, total, itemHash, MutateByteSlice(rootHash)) + if ok { + t.Errorf("Expected verification to fail for mutated root hash") + } + } +} diff --git a/merkle/types.go b/merkle/types.go new file mode 100644 index 00000000..a0c491a7 --- /dev/null +++ b/merkle/types.go @@ -0,0 +1,47 @@ +package merkle + +import ( + "encoding/binary" + "io" +) + +type Tree interface { + Size() (size int) + Height() (height int8) + Has(key []byte) (has bool) + Proof(key []byte) (value []byte, proof []byte, exists bool) // TODO make it return an index + Get(key []byte) (index int, value []byte, exists bool) + GetByIndex(index int) (key []byte, value []byte) + Set(key []byte, value []byte) (updated bool) + Remove(key []byte) (value []byte, removed bool) + HashWithCount() (hash []byte, count int) + Hash() (hash []byte) + Save() (hash []byte) + Load(hash []byte) + Copy() Tree + Iterate(func(key []byte, value []byte) (stop bool)) (stopped bool) + IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool) +} + +type Hasher interface { + Hash() []byte +} + +//----------------------------------------------------------------------- +// NOTE: these are duplicated from go-amino so we dont need go-amino as a dep + +func encodeByteSlice(w io.Writer, bz []byte) (err error) { + err = encodeUvarint(w, uint64(len(bz))) + if err != nil { + return + } + _, err = w.Write(bz) + return +} + +func encodeUvarint(w io.Writer, i uint64) (err error) { + var buf [10]byte + n := binary.PutUvarint(buf[:], i) + _, err = w.Write(buf[0:n]) + return +} From 4663ffdf087b571e5ccbc0df5ec6a6b8ad36cce1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 22 May 2018 12:46:15 -0400 Subject: [PATCH 2/6] add tmhash --- merkle/simple_map.go | 4 ++-- merkle/simple_map_test.go | 12 ++++++------ merkle/simple_tree.go | 6 +++--- tmhash/hash.go | 41 +++++++++++++++++++++++++++++++++++++++ tmhash/hash_test.go | 23 ++++++++++++++++++++++ 5 files changed, 75 insertions(+), 11 deletions(-) create mode 100644 tmhash/hash.go create mode 100644 tmhash/hash_test.go diff --git a/merkle/simple_map.go b/merkle/simple_map.go index cd38de76..41d1ccd5 100644 --- a/merkle/simple_map.go +++ b/merkle/simple_map.go @@ -2,7 +2,7 @@ package merkle import ( cmn "github.com/tendermint/tmlibs/common" - "golang.org/x/crypto/ripemd160" + "github.com/tendermint/go-crypto/tmhash" ) type SimpleMap struct { @@ -63,7 +63,7 @@ func (sm *SimpleMap) KVPairs() cmn.KVPairs { type KVPair cmn.KVPair func (kv KVPair) Hash() []byte { - hasher := ripemd160.New() + hasher := tmhash.New() err := encodeByteSlice(hasher, kv.Key) if err != nil { panic(err) diff --git a/merkle/simple_map_test.go b/merkle/simple_map_test.go index c9c87135..6e1004db 100644 --- a/merkle/simple_map_test.go +++ b/merkle/simple_map_test.go @@ -17,37 +17,37 @@ func TestSimpleMap(t *testing.T) { { db := NewSimpleMap() db.Set("key1", strHasher("value1")) - assert.Equal(t, "acdb4f121bc6f25041eb263ab463f1cd79236a32", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "3dafc06a52039d029be57c75c9d16356a4256ef4", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key1", strHasher("value2")) - assert.Equal(t, "b8cbf5adee8c524e14f531da9b49adbbbd66fffa", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "03eb5cfdff646bc4e80fec844e72fd248a1c6b2c", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key1", strHasher("value1")) db.Set("key2", strHasher("value2")) - assert.Equal(t, "1708aabc85bbe00242d3db8c299516aa54e48c38", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "acc3971eab8513171cc90ce8b74f368c38f9657d", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key2", strHasher("value2")) // NOTE: out of order db.Set("key1", strHasher("value1")) - assert.Equal(t, "1708aabc85bbe00242d3db8c299516aa54e48c38", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "acc3971eab8513171cc90ce8b74f368c38f9657d", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key1", strHasher("value1")) db.Set("key2", strHasher("value2")) db.Set("key3", strHasher("value3")) - assert.Equal(t, "e728afe72ce351eed6aca65c5f78da19b9a6e214", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "0cd117ad14e6cd22edcd9aa0d84d7063b54b862f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { db := NewSimpleMap() db.Set("key2", strHasher("value2")) // NOTE: out of order db.Set("key1", strHasher("value1")) db.Set("key3", strHasher("value3")) - assert.Equal(t, "e728afe72ce351eed6aca65c5f78da19b9a6e214", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") + assert.Equal(t, "0cd117ad14e6cd22edcd9aa0d84d7063b54b862f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } } diff --git a/merkle/simple_tree.go b/merkle/simple_tree.go index 9bdf52cb..3c31b133 100644 --- a/merkle/simple_tree.go +++ b/merkle/simple_tree.go @@ -25,11 +25,11 @@ For larger datasets, use IAVLTree. package merkle import ( - "golang.org/x/crypto/ripemd160" + "github.com/tendermint/go-crypto/tmhash" ) func SimpleHashFromTwoHashes(left []byte, right []byte) []byte { - var hasher = ripemd160.New() + var hasher = tmhash.New() err := encodeByteSlice(hasher, left) if err != nil { panic(err) @@ -68,7 +68,7 @@ func SimpleHashFromByteslices(bzs [][]byte) []byte { } func SimpleHashFromBytes(bz []byte) []byte { - hasher := ripemd160.New() + hasher := tmhash.New() hasher.Write(bz) return hasher.Sum(nil) } diff --git a/tmhash/hash.go b/tmhash/hash.go new file mode 100644 index 00000000..de69c406 --- /dev/null +++ b/tmhash/hash.go @@ -0,0 +1,41 @@ +package tmhash + +import ( + "crypto/sha256" + "hash" +) + +var ( + Size = 20 + BlockSize = sha256.BlockSize +) + +type sha256trunc struct { + sha256 hash.Hash +} + +func (h sha256trunc) Write(p []byte) (n int, err error) { + return h.sha256.Write(p) +} +func (h sha256trunc) Sum(b []byte) []byte { + shasum := h.sha256.Sum(b) + return shasum[:Size] +} + +func (h sha256trunc) Reset() { + h.sha256.Reset() +} + +func (h sha256trunc) Size() int { + return Size +} + +func (h sha256trunc) BlockSize() int { + return h.sha256.BlockSize() +} + +func New() hash.Hash { + return sha256trunc{ + sha256: sha256.New(), + } +} diff --git a/tmhash/hash_test.go b/tmhash/hash_test.go new file mode 100644 index 00000000..abf0247a --- /dev/null +++ b/tmhash/hash_test.go @@ -0,0 +1,23 @@ +package tmhash_test + +import ( + "crypto/sha256" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/go-crypto/tmhash" +) + +func TestHash(t *testing.T) { + testVector := []byte("abc") + hasher := tmhash.New() + hasher.Write(testVector) + bz := hasher.Sum(nil) + + hasher = sha256.New() + hasher.Write(testVector) + bz2 := hasher.Sum(nil) + bz2 = bz2[:20] + + assert.Equal(t, bz, bz2) +} From c2636c3c6b95cc314e6a7ddc079daf236a9a1a5a Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 22 May 2018 13:04:42 -0400 Subject: [PATCH 3/6] tmhash: add Sum function --- tmhash/hash.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tmhash/hash.go b/tmhash/hash.go index de69c406..9f684ce0 100644 --- a/tmhash/hash.go +++ b/tmhash/hash.go @@ -34,8 +34,15 @@ func (h sha256trunc) BlockSize() int { return h.sha256.BlockSize() } +// New returns a new hash.Hash. func New() hash.Hash { return sha256trunc{ sha256: sha256.New(), } } + +// Sum returns the first 20 bytes of SHA256 of the bz. +func Sum(bz []byte) []byte { + hash := sha256.Sum256(bz) + return hash[:Size] +} From 9f04935caa0ba104421c5c856bae92d5307e3869 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 22 May 2018 13:05:27 -0400 Subject: [PATCH 4/6] merkle: remove unused funcs. unexport simplemap. improv docs --- merkle/simple_map.go | 44 +++++++++------- merkle/simple_map_test.go | 17 ++++--- merkle/simple_proof.go | 18 +++++-- merkle/simple_tree.go | 101 +++++++++++++------------------------ merkle/simple_tree_test.go | 3 +- merkle/types.go | 2 + tmhash/hash.go | 2 +- 7 files changed, 88 insertions(+), 99 deletions(-) diff --git a/merkle/simple_map.go b/merkle/simple_map.go index 41d1ccd5..b073ecdd 100644 --- a/merkle/simple_map.go +++ b/merkle/simple_map.go @@ -5,23 +5,27 @@ import ( "github.com/tendermint/go-crypto/tmhash" ) -type SimpleMap struct { +// Merkle tree from a map. +// Leaves are `hash(key) | hash(value)`. +// Leaves are sorted before Merkle hashing. +type simpleMap struct { kvs cmn.KVPairs sorted bool } -func NewSimpleMap() *SimpleMap { - return &SimpleMap{ +func newSimpleMap() *simpleMap { + return &simpleMap{ kvs: nil, sorted: false, } } -func (sm *SimpleMap) Set(key string, value Hasher) { +// Set hashes the key and value and appends it to the kv pairs. +func (sm *simpleMap) Set(key string, value Hasher) { sm.sorted = false // Hash the key to blind it... why not? - khash := SimpleHashFromBytes([]byte(key)) + khash := tmhash.Sum([]byte(key)) // And the value is hashed too, so you can // check for equality with a cached value (say) @@ -34,14 +38,14 @@ func (sm *SimpleMap) Set(key string, value Hasher) { }) } -// Merkle root hash of items sorted by key +// Hash Merkle root hash of items sorted by key // (UNSTABLE: and by value too if duplicate key). -func (sm *SimpleMap) Hash() []byte { +func (sm *simpleMap) Hash() []byte { sm.Sort() return hashKVPairs(sm.kvs) } -func (sm *SimpleMap) Sort() { +func (sm *simpleMap) Sort() { if sm.sorted { return } @@ -50,7 +54,8 @@ func (sm *SimpleMap) Sort() { } // Returns a copy of sorted KVPairs. -func (sm *SimpleMap) KVPairs() cmn.KVPairs { +// NOTE these contain the hashed key and value. +func (sm *simpleMap) KVPairs() cmn.KVPairs { sm.Sort() kvs := make(cmn.KVPairs, len(sm.kvs)) copy(kvs, sm.kvs) @@ -60,25 +65,28 @@ func (sm *SimpleMap) KVPairs() cmn.KVPairs { //---------------------------------------- // A local extension to KVPair that can be hashed. -type KVPair cmn.KVPair +// XXX: key and value do not need to already be hashed - +// the kvpair ("abc", "def") would not give the same result +// as ("ab", "cdef") as we're using length-prefixing. +type kvPair cmn.KVPair -func (kv KVPair) Hash() []byte { +func (kv kvPair) Hash() []byte { hasher := tmhash.New() err := encodeByteSlice(hasher, kv.Key) if err != nil { - panic(err) - } + panic(err) + } err = encodeByteSlice(hasher, kv.Value) if err != nil { - panic(err) - } + panic(err) + } return hasher.Sum(nil) } func hashKVPairs(kvs cmn.KVPairs) []byte { - kvsH := make([]Hasher, 0, len(kvs)) - for _, kvp := range kvs { - kvsH = append(kvsH, KVPair(kvp)) + kvsH := make([]Hasher, len(kvs)) + for i, kvp := range kvs { + kvsH[i] = kvPair(kvp) } return SimpleHashFromHashers(kvsH) } diff --git a/merkle/simple_map_test.go b/merkle/simple_map_test.go index 6e1004db..9d0c25a2 100644 --- a/merkle/simple_map_test.go +++ b/merkle/simple_map_test.go @@ -5,46 +5,49 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/tendermint/go-crypto/tmhash" ) type strHasher string func (str strHasher) Hash() []byte { - return SimpleHashFromBytes([]byte(str)) + h := tmhash.New() + h.Write([]byte(str)) + return h.Sum(nil) } func TestSimpleMap(t *testing.T) { { - db := NewSimpleMap() + db := newSimpleMap() db.Set("key1", strHasher("value1")) assert.Equal(t, "3dafc06a52039d029be57c75c9d16356a4256ef4", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { - db := NewSimpleMap() + db := newSimpleMap() db.Set("key1", strHasher("value2")) assert.Equal(t, "03eb5cfdff646bc4e80fec844e72fd248a1c6b2c", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { - db := NewSimpleMap() + db := newSimpleMap() db.Set("key1", strHasher("value1")) db.Set("key2", strHasher("value2")) assert.Equal(t, "acc3971eab8513171cc90ce8b74f368c38f9657d", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { - db := NewSimpleMap() + db := newSimpleMap() db.Set("key2", strHasher("value2")) // NOTE: out of order db.Set("key1", strHasher("value1")) assert.Equal(t, "acc3971eab8513171cc90ce8b74f368c38f9657d", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { - db := NewSimpleMap() + db := newSimpleMap() db.Set("key1", strHasher("value1")) db.Set("key2", strHasher("value2")) db.Set("key3", strHasher("value3")) assert.Equal(t, "0cd117ad14e6cd22edcd9aa0d84d7063b54b862f", fmt.Sprintf("%x", db.Hash()), "Hash didn't match") } { - db := NewSimpleMap() + db := newSimpleMap() db.Set("key2", strHasher("value2")) // NOTE: out of order db.Set("key1", strHasher("value1")) db.Set("key3", strHasher("value3")) diff --git a/merkle/simple_proof.go b/merkle/simple_proof.go index ca6ccf37..f52d1ad9 100644 --- a/merkle/simple_proof.go +++ b/merkle/simple_proof.go @@ -5,10 +5,12 @@ import ( "fmt" ) +// SimpleProof represents a simple merkle proof. type SimpleProof struct { Aunts [][]byte `json:"aunts"` // Hashes from leaf's sibling to a root's child. } +// SimpleProofsFromHashers computes inclusion proof for given items. // proofs[0] is the proof for items[0]. func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleProof) { trails, rootSPN := trailsFromHashers(items) @@ -22,8 +24,11 @@ func SimpleProofsFromHashers(items []Hasher) (rootHash []byte, proofs []*SimpleP return } +// SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values +// in the underlying key-value pairs. +// The keys are sorted before the proofs are computed. func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs []*SimpleProof) { - sm := NewSimpleMap() + sm := newSimpleMap() for k, v := range m { sm.Set(k, v) } @@ -31,7 +36,7 @@ func SimpleProofsFromMap(m map[string]Hasher) (rootHash []byte, proofs []*Simple kvs := sm.kvs kvsH := make([]Hasher, 0, len(kvs)) for _, kvp := range kvs { - kvsH = append(kvsH, KVPair(kvp)) + kvsH = append(kvsH, kvPair(kvp)) } return SimpleProofsFromHashers(kvsH) } @@ -43,10 +48,13 @@ func (sp *SimpleProof) Verify(index int, total int, leafHash []byte, rootHash [] return computedHash != nil && bytes.Equal(computedHash, rootHash) } +// String implements the stringer interface for SimpleProof. +// It is a wrapper around StringIndented. func (sp *SimpleProof) String() string { return sp.StringIndented("") } +// StringIndented generates a canonical string representation of a SimpleProof. func (sp *SimpleProof) StringIndented(indent string) string { return fmt.Sprintf(`SimpleProof{ %s Aunts: %X @@ -90,7 +98,7 @@ func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][ } } -// Helper structure to construct merkle proof. +// SimpleProofNode is a helper structure to construct merkle proof. // The node and the tree is thrown away afterwards. // Exactly one of node.Left and node.Right is nil, unless node is the root, in which case both are nil. // node.Parent.Hash = hash(node.Hash, node.Right.Hash) or @@ -102,8 +110,8 @@ type SimpleProofNode struct { Right *SimpleProofNode // Right sibling (only one of Left,Right is set) } -// Starting from a leaf SimpleProofNode, FlattenAunts() will return -// the inner hashes for the item corresponding to the leaf. +// FlattenAunts will return the inner hashes for the item corresponding to the leaf, +// starting from a leaf SimpleProofNode. func (spn *SimpleProofNode) FlattenAunts() [][]byte { // Nonrecursive impl. innerHashes := [][]byte{} diff --git a/merkle/simple_tree.go b/merkle/simple_tree.go index 3c31b133..c23f8426 100644 --- a/merkle/simple_tree.go +++ b/merkle/simple_tree.go @@ -1,47 +1,49 @@ -/* -Computes a deterministic minimal height merkle tree hash. -If the number of items is not a power of two, some leaves -will be at different levels. Tries to keep both sides of -the tree the same size, but the left may be one greater. - -Use this for short deterministic trees, such as the validator list. -For larger datasets, use IAVLTree. - - * - / \ - / \ - / \ - / \ - * * - / \ / \ - / \ / \ - / \ / \ - * * * h6 - / \ / \ / \ - h0 h1 h2 h3 h4 h5 - -*/ - package merkle import ( "github.com/tendermint/go-crypto/tmhash" ) -func SimpleHashFromTwoHashes(left []byte, right []byte) []byte { +// SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). +func SimpleHashFromTwoHashes(left, right []byte) []byte { var hasher = tmhash.New() err := encodeByteSlice(hasher, left) if err != nil { - panic(err) - } + panic(err) + } err = encodeByteSlice(hasher, right) if err != nil { - panic(err) - } + panic(err) + } return hasher.Sum(nil) } -func SimpleHashFromHashes(hashes [][]byte) []byte { +// SimpleHashFromHashers computes a Merkle tree from items that can be hashed. +func SimpleHashFromHashers(items []Hasher) []byte { + hashes := make([][]byte, len(items)) + for i, item := range items { + hash := item.Hash() + hashes[i] = hash + } + return simpleHashFromHashes(hashes) +} + +// SimpleHashFromMap computes a Merkle tree from sorted map. +// Like calling SimpleHashFromHashers with +// `item = []byte(Hash(key) | Hash(value))`, +// sorted by `item`. +func SimpleHashFromMap(m map[string]Hasher) []byte { + sm := newSimpleMap() + for k, v := range m { + sm.Set(k, v) + } + return sm.Hash() +} + +//---------------------------------------------------------------- + +// Expects hashes! +func simpleHashFromHashes(hashes [][]byte) []byte { // Recursive impl. switch len(hashes) { case 0: @@ -49,43 +51,8 @@ func SimpleHashFromHashes(hashes [][]byte) []byte { case 1: return hashes[0] default: - left := SimpleHashFromHashes(hashes[:(len(hashes)+1)/2]) - right := SimpleHashFromHashes(hashes[(len(hashes)+1)/2:]) + left := simpleHashFromHashes(hashes[:(len(hashes)+1)/2]) + right := simpleHashFromHashes(hashes[(len(hashes)+1)/2:]) return SimpleHashFromTwoHashes(left, right) } } - -// NOTE: Do not implement this, use SimpleHashFromByteslices instead. -// type Byteser interface { Bytes() []byte } -// func SimpleHashFromBytesers(items []Byteser) []byte { ... } - -func SimpleHashFromByteslices(bzs [][]byte) []byte { - hashes := make([][]byte, len(bzs)) - for i, bz := range bzs { - hashes[i] = SimpleHashFromBytes(bz) - } - return SimpleHashFromHashes(hashes) -} - -func SimpleHashFromBytes(bz []byte) []byte { - hasher := tmhash.New() - hasher.Write(bz) - return hasher.Sum(nil) -} - -func SimpleHashFromHashers(items []Hasher) []byte { - hashes := make([][]byte, len(items)) - for i, item := range items { - hash := item.Hash() - hashes[i] = hash - } - return SimpleHashFromHashes(hashes) -} - -func SimpleHashFromMap(m map[string]Hasher) []byte { - sm := NewSimpleMap() - for k, v := range m { - sm.Set(k, v) - } - return sm.Hash() -} diff --git a/merkle/simple_tree_test.go b/merkle/simple_tree_test.go index 8c4ed01f..db8e3d7f 100644 --- a/merkle/simple_tree_test.go +++ b/merkle/simple_tree_test.go @@ -7,6 +7,7 @@ import ( . "github.com/tendermint/tmlibs/test" "testing" + "github.com/tendermint/go-crypto/tmhash" ) type testItem []byte @@ -21,7 +22,7 @@ func TestSimpleProof(t *testing.T) { items := make([]Hasher, total) for i := 0; i < total; i++ { - items[i] = testItem(cmn.RandBytes(32)) + items[i] = testItem(cmn.RandBytes(tmhash.Size)) } rootHash := SimpleHashFromHashers(items) diff --git a/merkle/types.go b/merkle/types.go index a0c491a7..ddc42065 100644 --- a/merkle/types.go +++ b/merkle/types.go @@ -5,6 +5,7 @@ import ( "io" ) +// Tree is a Merkle tree interface. type Tree interface { Size() (size int) Height() (height int8) @@ -23,6 +24,7 @@ type Tree interface { IterateRange(start []byte, end []byte, ascending bool, fx func(key []byte, value []byte) (stop bool)) (stopped bool) } +// Hasher represents a hashable piece of data which can be hashed in the Tree. type Hasher interface { Hash() []byte } diff --git a/tmhash/hash.go b/tmhash/hash.go index 9f684ce0..1b29d868 100644 --- a/tmhash/hash.go +++ b/tmhash/hash.go @@ -5,7 +5,7 @@ import ( "hash" ) -var ( +const ( Size = 20 BlockSize = sha256.BlockSize ) From 862d3c342a03861c74cfbc649439cc81ceff5396 Mon Sep 17 00:00:00 2001 From: Liamsi Date: Wed, 30 May 2018 17:28:20 +0100 Subject: [PATCH 5/6] commit doc.go --- merkle/doc.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 merkle/doc.go diff --git a/merkle/doc.go b/merkle/doc.go new file mode 100644 index 00000000..da65dd85 --- /dev/null +++ b/merkle/doc.go @@ -0,0 +1,31 @@ +/* +Package merkle computes a deterministic minimal height Merkle tree hash. +If the number of items is not a power of two, some leaves +will be at different levels. Tries to keep both sides of +the tree the same size, but the left may be one greater. + +Use this for short deterministic trees, such as the validator list. +For larger datasets, use IAVLTree. + +Be aware that the current implementation by itself does not prevent +second pre-image attacks. Hence, use this library with caution. +Otherwise you might run into similar issues as, e.g., in early Bitcoin: +https://bitcointalk.org/?topic=102395 + + * + / \ + / \ + / \ + / \ + * * + / \ / \ + / \ / \ + / \ / \ + * * * h6 + / \ / \ / \ + h0 h1 h2 h3 h4 h5 + +TODO(ismail): add 2nd pre-image protection or clarify further on how we use this and why this secure. + +*/ +package merkle \ No newline at end of file From 52bd867fd918513e7cab9d57cf4fdf9c78396a03 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 30 May 2018 17:50:17 -0400 Subject: [PATCH 6/6] merkle: use amino for byteslice encoding --- merkle/simple_map.go | 15 +++++++-------- merkle/simple_map_test.go | 4 +--- merkle/types.go | 19 ++++--------------- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/merkle/simple_map.go b/merkle/simple_map.go index b073ecdd..cde5924f 100644 --- a/merkle/simple_map.go +++ b/merkle/simple_map.go @@ -1,8 +1,8 @@ package merkle import ( - cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/go-crypto/tmhash" + cmn "github.com/tendermint/tmlibs/common" ) // Merkle tree from a map. @@ -65,21 +65,20 @@ func (sm *simpleMap) KVPairs() cmn.KVPairs { //---------------------------------------- // A local extension to KVPair that can be hashed. -// XXX: key and value do not need to already be hashed - -// the kvpair ("abc", "def") would not give the same result -// as ("ab", "cdef") as we're using length-prefixing. +// Key and value are length prefixed and concatenated, +// then hashed. type kvPair cmn.KVPair func (kv kvPair) Hash() []byte { hasher := tmhash.New() err := encodeByteSlice(hasher, kv.Key) if err != nil { - panic(err) - } + panic(err) + } err = encodeByteSlice(hasher, kv.Value) if err != nil { - panic(err) - } + panic(err) + } return hasher.Sum(nil) } diff --git a/merkle/simple_map_test.go b/merkle/simple_map_test.go index 9d0c25a2..a89289a8 100644 --- a/merkle/simple_map_test.go +++ b/merkle/simple_map_test.go @@ -11,9 +11,7 @@ import ( type strHasher string func (str strHasher) Hash() []byte { - h := tmhash.New() - h.Write([]byte(str)) - return h.Sum(nil) + return tmhash.Sum([]byte(str)) } func TestSimpleMap(t *testing.T) { diff --git a/merkle/types.go b/merkle/types.go index ddc42065..2fcb3f39 100644 --- a/merkle/types.go +++ b/merkle/types.go @@ -1,8 +1,9 @@ package merkle import ( - "encoding/binary" "io" + + amino "github.com/tendermint/go-amino" ) // Tree is a Merkle tree interface. @@ -30,20 +31,8 @@ type Hasher interface { } //----------------------------------------------------------------------- -// NOTE: these are duplicated from go-amino so we dont need go-amino as a dep +// Uvarint length prefixed byteslice func encodeByteSlice(w io.Writer, bz []byte) (err error) { - err = encodeUvarint(w, uint64(len(bz))) - if err != nil { - return - } - _, err = w.Write(bz) - return -} - -func encodeUvarint(w io.Writer, i uint64) (err error) { - var buf [10]byte - n := binary.PutUvarint(buf[:], i) - _, err = w.Write(buf[0:n]) - return + return amino.EncodeByteSlice(w, bz) }