Unified deterministic merkle tree computation. Now we have 2 types of
trees... IAVL (nondeterministic) and Simple (deterministic)
This commit is contained in:
parent
c035a383b7
commit
a0cc186c75
|
@ -0,0 +1,173 @@
|
|||
/*
|
||||
Computes a deterministic minimal height merkle tree hash.
|
||||
If the number of items is not a power of two, some leaves
|
||||
will be at different levels, leaning left.
|
||||
|
||||
Use this for shorter deterministic trees, such as the validator list.
|
||||
For larger datasets, use IAVLTree.
|
||||
|
||||
*
|
||||
/ \
|
||||
/ \
|
||||
/ \
|
||||
/ \
|
||||
* *
|
||||
/ \ / \
|
||||
/ \ / \
|
||||
/ \ / \
|
||||
* * * h6
|
||||
/ \ / \ / \
|
||||
h0 h1 h2 h3 h4 h5
|
||||
|
||||
*/
|
||||
|
||||
package merkle
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
|
||||
. "github.com/tendermint/tendermint/binary"
|
||||
)
|
||||
|
||||
func HashFromTwoHashes(left []byte, right []byte) []byte {
|
||||
var n int64
|
||||
var err error
|
||||
var hasher = sha256.New()
|
||||
WriteByteSlice(hasher, left, &n, &err)
|
||||
WriteByteSlice(hasher, right, &n, &err)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return hasher.Sum(nil)
|
||||
}
|
||||
|
||||
func HashFromHashes(hashes [][]byte) []byte {
|
||||
// Recursive impl.
|
||||
switch len(hashes) {
|
||||
case 0:
|
||||
panic("Cannot call HashFromHashes() with 0 length arg")
|
||||
case 1:
|
||||
return hashes[0]
|
||||
default:
|
||||
left := HashFromHashes(hashes[:(len(hashes)+1)/2])
|
||||
right := HashFromHashes(hashes[(len(hashes)+1)/2:])
|
||||
return HashFromTwoHashes(left, right)
|
||||
}
|
||||
}
|
||||
|
||||
// Convenience for HashFromHashes.
|
||||
func HashFromBinaries(items []Binary) []byte {
|
||||
hashes := [][]byte{}
|
||||
for _, item := range items {
|
||||
hasher := sha256.New()
|
||||
_, err := item.WriteTo(hasher)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
hash := hasher.Sum(nil)
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
return HashFromHashes(hashes)
|
||||
}
|
||||
|
||||
// Convenience for HashFromHashes.
|
||||
func HashFromHashables(items []Hashable) []byte {
|
||||
hashes := [][]byte{}
|
||||
for _, item := range items {
|
||||
hash := item.Hash()
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
return HashFromHashes(hashes)
|
||||
}
|
||||
|
||||
type HashTrail struct {
|
||||
Hash []byte
|
||||
Parent *HashTrail
|
||||
Left *HashTrail
|
||||
Right *HashTrail
|
||||
}
|
||||
|
||||
func (ht *HashTrail) Flatten() [][]byte {
|
||||
// Nonrecursive impl.
|
||||
trail := [][]byte{}
|
||||
for ht != nil {
|
||||
if ht.Left != nil {
|
||||
trail = append(trail, ht.Left.Hash)
|
||||
} else if ht.Right != nil {
|
||||
trail = append(trail, ht.Right.Hash)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
ht = ht.Parent
|
||||
}
|
||||
return trail
|
||||
}
|
||||
|
||||
// returned trails[0].Hash is the leaf hash.
|
||||
// trails[0].Parent.Hash is the hash above that, etc.
|
||||
func HashTrailsFromHashables(items []Hashable) (trails []*HashTrail, root *HashTrail) {
|
||||
// Recursive impl.
|
||||
switch len(items) {
|
||||
case 0:
|
||||
panic("Cannot call HashTrailsFromHashables() with 0 length arg")
|
||||
case 1:
|
||||
trail := &HashTrail{items[0].Hash(), nil, nil, nil}
|
||||
return []*HashTrail{trail}, trail
|
||||
default:
|
||||
lefts, leftRoot := HashTrailsFromHashables(items[:(len(items)+1)/2])
|
||||
rights, rightRoot := HashTrailsFromHashables(items[(len(items)+1)/2:])
|
||||
rootHash := HashFromTwoHashes(leftRoot.Hash, rightRoot.Hash)
|
||||
root := &HashTrail{rootHash, nil, nil, nil}
|
||||
leftRoot.Parent = root
|
||||
leftRoot.Right = rightRoot
|
||||
rightRoot.Parent = root
|
||||
rightRoot.Left = leftRoot
|
||||
return append(lefts, rights...), root
|
||||
}
|
||||
}
|
||||
|
||||
// Ensures that leafHash is part of rootHash.
|
||||
func VerifyHashTrail(index uint, total uint, leafHash []byte, trail [][]byte, rootHash []byte) bool {
|
||||
computedRoot := ComputeRootFromTrail(index, total, leafHash, trail)
|
||||
if computedRoot == nil {
|
||||
return false
|
||||
}
|
||||
return bytes.Equal(computedRoot, rootHash)
|
||||
}
|
||||
|
||||
// Use the leafHash and trail to get the root merkle hash.
|
||||
// If the length of the trail slice isn't exactly correct, the result is nil.
|
||||
func ComputeRootFromTrail(index uint, total uint, leafHash []byte, trail [][]byte) []byte {
|
||||
// Recursive impl.
|
||||
if index >= total {
|
||||
return nil
|
||||
}
|
||||
switch total {
|
||||
case 0:
|
||||
panic("Cannot call ComputeRootFromTrail() with 0 total")
|
||||
case 1:
|
||||
if len(trail) != 0 {
|
||||
return nil
|
||||
}
|
||||
return leafHash
|
||||
default:
|
||||
if len(trail) == 0 {
|
||||
return nil
|
||||
}
|
||||
numLeft := (total + 1) / 2
|
||||
if index < numLeft {
|
||||
leftRoot := ComputeRootFromTrail(index, numLeft, leafHash, trail[:len(trail)-1])
|
||||
if leftRoot == nil {
|
||||
return nil
|
||||
}
|
||||
return HashFromTwoHashes(leftRoot, trail[len(trail)-1])
|
||||
} else {
|
||||
rightRoot := ComputeRootFromTrail(index-numLeft, total-numLeft, leafHash, trail[:len(trail)-1])
|
||||
if rightRoot == nil {
|
||||
return nil
|
||||
}
|
||||
return HashFromTwoHashes(trail[len(trail)-1], rightRoot)
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,74 @@
|
|||
package merkle
|
||||
|
||||
import (
|
||||
. "github.com/tendermint/tendermint/common"
|
||||
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type testItem []byte
|
||||
|
||||
func (tI testItem) Hash() []byte {
|
||||
return []byte(tI)
|
||||
}
|
||||
|
||||
func TestMerkleTrails(t *testing.T) {
|
||||
|
||||
numItems := uint(100)
|
||||
|
||||
items := make([]Hashable, numItems)
|
||||
for i := uint(0); i < numItems; i++ {
|
||||
items[i] = testItem(RandBytes(32))
|
||||
}
|
||||
|
||||
root := HashFromHashables(items)
|
||||
|
||||
trails, rootTrail := HashTrailsFromHashables(items)
|
||||
|
||||
// Assert that HashFromHashables and HashTrailsFromHashables are compatible.
|
||||
if !bytes.Equal(root, rootTrail.Hash) {
|
||||
t.Errorf("Root mismatch:\n%X vs\n%X", root, rootTrail.Hash)
|
||||
}
|
||||
|
||||
// For each item, check the trail.
|
||||
for i, item := range items {
|
||||
itemHash := item.Hash()
|
||||
flatTrail := trails[i].Flatten()
|
||||
|
||||
// Verify success
|
||||
ok := VerifyHashTrail(uint(i), numItems, itemHash, flatTrail, root)
|
||||
if !ok {
|
||||
t.Errorf("Verification failed for index %v.", i)
|
||||
}
|
||||
|
||||
// Wrong item index should make it fail
|
||||
ok = VerifyHashTrail(uint(i)+1, numItems, itemHash, flatTrail, root)
|
||||
if ok {
|
||||
t.Errorf("Expected verification to fail for wrong index %v.", i)
|
||||
}
|
||||
|
||||
// Trail too long should make it fail
|
||||
trail2 := append(flatTrail, RandBytes(32))
|
||||
ok = VerifyHashTrail(uint(i), numItems, itemHash, trail2, root)
|
||||
if ok {
|
||||
t.Errorf("Expected verification to fail for wrong trail length.")
|
||||
}
|
||||
|
||||
// Trail too short should make it fail
|
||||
trail2 = flatTrail[:len(flatTrail)-1]
|
||||
ok = VerifyHashTrail(uint(i), numItems, itemHash, trail2, root)
|
||||
if ok {
|
||||
t.Errorf("Expected verification to fail for wrong trail length.")
|
||||
}
|
||||
|
||||
// Mutating the itemHash should make it fail.
|
||||
itemHash2 := make([]byte, len(itemHash))
|
||||
copy(itemHash2, itemHash)
|
||||
itemHash2[0] += byte(0x01)
|
||||
ok = VerifyHashTrail(uint(i), numItems, itemHash2, flatTrail, root)
|
||||
if ok {
|
||||
t.Errorf("Expected verification to fail for mutated leaf hash")
|
||||
}
|
||||
}
|
||||
}
|
222
merkle/util.go
222
merkle/util.go
|
@ -1,230 +1,10 @@
|
|||
package merkle
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
|
||||
. "github.com/tendermint/tendermint/binary"
|
||||
)
|
||||
|
||||
func HashFromTwoHashes(left []byte, right []byte) []byte {
|
||||
var n int64
|
||||
var err error
|
||||
var hasher = sha256.New()
|
||||
WriteByteSlice(hasher, left, &n, &err)
|
||||
WriteByteSlice(hasher, right, &n, &err)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return hasher.Sum(nil)
|
||||
}
|
||||
|
||||
/*
|
||||
Computes a deterministic minimal height merkle tree hash.
|
||||
If the number of items is not a power of two, some leaves
|
||||
will be at different levels.
|
||||
|
||||
*
|
||||
/ \
|
||||
/ \
|
||||
/ \
|
||||
/ \
|
||||
* *
|
||||
/ \ / \
|
||||
/ \ / \
|
||||
/ \ / \
|
||||
* h2 * *
|
||||
/ \ / \ / \
|
||||
h0 h1 h3 h4 h5 h6
|
||||
|
||||
*/
|
||||
func HashFromHashes(hashes [][]byte) []byte {
|
||||
switch len(hashes) {
|
||||
case 0:
|
||||
return nil
|
||||
case 1:
|
||||
return hashes[0]
|
||||
default:
|
||||
left := HashFromHashes(hashes[:len(hashes)/2])
|
||||
right := HashFromHashes(hashes[len(hashes)/2:])
|
||||
return HashFromTwoHashes(left, right)
|
||||
}
|
||||
}
|
||||
|
||||
// Convenience for HashFromHashes.
|
||||
func HashFromBinaries(items []Binary) []byte {
|
||||
hashes := [][]byte{}
|
||||
for _, item := range items {
|
||||
hasher := sha256.New()
|
||||
_, err := item.WriteTo(hasher)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
hash := hasher.Sum(nil)
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
return HashFromHashes(hashes)
|
||||
}
|
||||
|
||||
// Convenience for HashFromHashes.
|
||||
func HashFromHashables(items []Hashable) []byte {
|
||||
hashes := [][]byte{}
|
||||
for _, item := range items {
|
||||
hash := item.Hash()
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
return HashFromHashes(hashes)
|
||||
}
|
||||
|
||||
/*
|
||||
Calculates an array of hashes, useful for deriving hash trails.
|
||||
|
||||
7
|
||||
/ \
|
||||
/ \
|
||||
/ \
|
||||
/ \
|
||||
3 11
|
||||
/ \ / \
|
||||
/ \ / \
|
||||
/ \ / \
|
||||
1 5 9 13
|
||||
/ \ / \ / \ / \
|
||||
0 2 4 6 8 10 12 14
|
||||
h0 h1 h2 h3 h4 h5 h6 h7
|
||||
|
||||
(diagram and idea borrowed from libswift)
|
||||
|
||||
The hashes provided get assigned to even indices.
|
||||
The derived merkle hashes get assigned to odd indices.
|
||||
If "hashes" is not of length power of 2, it is padded
|
||||
with blank (zeroed) hashes.
|
||||
*/
|
||||
func HashTreeFromHashes(hashes [][]byte) [][]byte {
|
||||
|
||||
// Make length of "hashes" a power of 2
|
||||
hashesLen := uint32(len(hashes))
|
||||
fullLen := uint32(1)
|
||||
for {
|
||||
if fullLen >= hashesLen {
|
||||
break
|
||||
} else {
|
||||
fullLen <<= 1
|
||||
}
|
||||
}
|
||||
blank := make([]byte, len(hashes[0]))
|
||||
for i := hashesLen; i < fullLen; i++ {
|
||||
hashes = append(hashes, blank)
|
||||
}
|
||||
|
||||
// The result is twice the length minus one.
|
||||
res := make([][]byte, len(hashes)*2-1)
|
||||
for i, hash := range hashes {
|
||||
res[i*2] = hash
|
||||
}
|
||||
|
||||
// Fill all the hashes recursively.
|
||||
fillTreeRoot(res, 0, len(res)-1)
|
||||
return res
|
||||
}
|
||||
|
||||
// Fill in the blanks.
|
||||
func fillTreeRoot(res [][]byte, start, end int) []byte {
|
||||
if start == end {
|
||||
return res[start]
|
||||
} else {
|
||||
mid := (start + end) / 2
|
||||
left := fillTreeRoot(res, start, mid-1)
|
||||
right := fillTreeRoot(res, mid+1, end)
|
||||
root := HashFromTwoHashes(left, right)
|
||||
res[mid] = root
|
||||
return root
|
||||
}
|
||||
}
|
||||
|
||||
// Convenience for HashTreeFromHashes.
|
||||
func HashTreeFromHashables(items []Hashable) [][]byte {
|
||||
hashes := [][]byte{}
|
||||
for _, item := range items {
|
||||
hash := item.Hash()
|
||||
hashes = append(hashes, hash)
|
||||
}
|
||||
return HashTreeFromHashes(hashes)
|
||||
}
|
||||
|
||||
/*
|
||||
Given the original index of an item,
|
||||
(e.g. for h5 in the diagram above, the index is 5, not 10)
|
||||
returns a trail of hashes, which along with the index can be
|
||||
used to calculate the merkle root.
|
||||
|
||||
See VerifyHashTrailForIndex()
|
||||
*/
|
||||
func HashTrailForIndex(hashTree [][]byte, index int) [][]byte {
|
||||
trail := [][]byte{}
|
||||
index *= 2
|
||||
|
||||
// We start from the leaf layer and work our way up.
|
||||
// Notice the indices in the diagram:
|
||||
// 0 2 4 ... offset 0, stride 2
|
||||
// 1 5 9 ... offset 1, stride 4
|
||||
// 3 11 19 ... offset 3, stride 8
|
||||
// 7 23 39 ... offset 7, stride 16 etc.
|
||||
|
||||
offset := 0
|
||||
stride := 2
|
||||
|
||||
for {
|
||||
// Calculate sibling of index.
|
||||
var next int
|
||||
if ((index-offset)/stride)%2 == 0 {
|
||||
next = index + stride
|
||||
} else {
|
||||
next = index - stride
|
||||
}
|
||||
if next >= len(hashTree) {
|
||||
break
|
||||
}
|
||||
// Insert sibling hash to trail.
|
||||
trail = append(trail, hashTree[next])
|
||||
|
||||
index = (index + next) / 2
|
||||
offset += stride / 2
|
||||
stride *= 2
|
||||
}
|
||||
|
||||
return trail
|
||||
}
|
||||
|
||||
// Ensures that leafHash is part of rootHash.
|
||||
func VerifyHashTrailForIndex(index int, leafHash []byte, trail [][]byte, rootHash []byte) bool {
|
||||
index *= 2
|
||||
offset := 0
|
||||
stride := 2
|
||||
|
||||
tempHash := make([]byte, len(leafHash))
|
||||
copy(tempHash, leafHash)
|
||||
|
||||
for i := 0; i < len(trail); i++ {
|
||||
var next int
|
||||
if ((index-offset)/stride)%2 == 0 {
|
||||
next = index + stride
|
||||
tempHash = HashFromTwoHashes(tempHash, trail[i])
|
||||
} else {
|
||||
next = index - stride
|
||||
tempHash = HashFromTwoHashes(trail[i], tempHash)
|
||||
}
|
||||
index = (index + next) / 2
|
||||
offset += stride / 2
|
||||
stride *= 2
|
||||
}
|
||||
|
||||
return bytes.Equal(rootHash, tempHash)
|
||||
}
|
||||
|
||||
//-----------------------------------------------------------------------------
|
||||
|
||||
// Prints the in-memory children recursively.
|
||||
func PrintIAVLNode(node *IAVLNode) {
|
||||
fmt.Println("==== NODE")
|
||||
if node != nil {
|
||||
|
|
|
@ -1,35 +0,0 @@
|
|||
package merkle
|
||||
|
||||
import (
|
||||
. "github.com/tendermint/tendermint/common"
|
||||
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TODO: Actually test. All this does is help debug.
|
||||
// consensus/part_set tests some of this functionality.
|
||||
func TestHashTreeMerkleTrail(t *testing.T) {
|
||||
|
||||
numHashes := 5
|
||||
|
||||
// Make some fake "hashes".
|
||||
hashes := make([][]byte, numHashes)
|
||||
for i := 0; i < numHashes; i++ {
|
||||
hashes[i] = RandBytes(32)
|
||||
t.Logf("hash %v\t%X\n", i, hashes[i])
|
||||
}
|
||||
|
||||
hashTree := HashTreeFromHashes(hashes)
|
||||
for i := 0; i < len(hashTree); i++ {
|
||||
t.Logf("tree %v\t%X\n", i, hashTree[i])
|
||||
}
|
||||
|
||||
for i := 0; i < numHashes; i++ {
|
||||
t.Logf("trail %v\n", i)
|
||||
trail := HashTrailForIndex(hashTree, i)
|
||||
for j := 0; j < len(trail); j++ {
|
||||
t.Logf("index: %v, hash: %X\n", j, trail[j])
|
||||
}
|
||||
}
|
||||
|
||||
}
|
Loading…
Reference in New Issue