types: add ics23-tendermint to sdk (#6487)

* add ics23-tendermint to sdk

* fix errors and prealloc

* move ics-23 to internal

* minor changes and move maps to internal

* remvoe some usage of tendermint merkle

* fix linter

* dont use named returns

* move internal to store/rootmulti

* fix imports

Co-authored-by: Federico Kunze <31522760+fedekunze@users.noreply.github.com>
This commit is contained in:
Marko 2020-06-26 18:14:40 +02:00 committed by GitHub
parent 98a3645d0f
commit 3df4dd0cd9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 557 additions and 58 deletions

1
go.mod
View File

@ -9,7 +9,6 @@ require (
github.com/btcsuite/btcd v0.20.1-beta
github.com/btcsuite/btcutil v1.0.2
github.com/confio/ics23-iavl v0.6.0
github.com/confio/ics23-tendermint v0.6.1
github.com/confio/ics23/go v0.0.0-20200604202538-6e2c36a74465
github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d
github.com/cosmos/ledger-cosmos-go v0.11.1

2
go.sum
View File

@ -86,8 +86,6 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/confio/ics23-iavl v0.6.0 h1:vVRCuVaP38FCw1kTeEdFuGuiY+2vAGTBQoH7Zxkq/ws=
github.com/confio/ics23-iavl v0.6.0/go.mod h1:mmXAxD1vWoO0VP8YHu6mM1QHGv71NQqa1iSVm4HeKcY=
github.com/confio/ics23-tendermint v0.6.1 h1:cnakVCG9+SltTJnwh43Z4uhWFEr3V0t3PF1zM9mewTo=
github.com/confio/ics23-tendermint v0.6.1/go.mod h1:QOu6qLeiLIFMaLpc9R3QboIiw+mMA1N2Nc1Qnn7P6xc=
github.com/confio/ics23/go v0.0.0-20200323120010-7d9a00f0a2fa/go.mod h1:W1I3XC8d9N8OTu/ct5VJ84ylcOunZwMXsWkd27nvVts=
github.com/confio/ics23/go v0.0.0-20200604202538-6e2c36a74465 h1:tyK54ttJ14HaHaKjB6sQqkZaUSe/LUXKHjfgJNtcj20=
github.com/confio/ics23/go v0.0.0-20200604202538-6e2c36a74465/go.mod h1:W1I3XC8d9N8OTu/ct5VJ84ylcOunZwMXsWkd27nvVts=

View File

@ -1,4 +1,4 @@
package rootmulti
package maps
import (
"bytes"
@ -24,9 +24,9 @@ func newMerkleMap() *merkleMap {
}
}
// set creates a kv.Pair from the provided key and value. The value is hashed prior
// to creating a kv.Pair. The created kv.Pair is appended to the merkleMap's slice
// of kv.Pairs. Whenever called, the merkleMap must be resorted.
// Set creates a kv.Pair from the provided key and value. The value is hashed prior
// to creating a kv.Pair. The created kv.Pair is appended to the MerkleMap's slice
// of kv.Pairs. Whenever called, the MerkleMap must be resorted.
func (sm *merkleMap) set(key string, value []byte) {
sm.sorted = false
@ -40,7 +40,7 @@ func (sm *merkleMap) set(key string, value []byte) {
})
}
// hash returns the merkle root of items sorted by key. Note, it is unstable.
// Hash returns the merkle root of items sorted by key. Note, it is unstable.
func (sm *merkleMap) hash() []byte {
sm.sort()
return hashKVPairs(sm.kvs)
@ -107,19 +107,19 @@ func hashKVPairs(kvs kv.Pairs) []byte {
// Leaves are `hash(key) | hash(value)`.
// Leaves are sorted before Merkle hashing.
type simpleMap struct {
kvs kv.Pairs
Kvs kv.Pairs
sorted bool
}
func newSimpleMap() *simpleMap {
return &simpleMap{
kvs: nil,
Kvs: nil,
sorted: false,
}
}
// Set creates a kv pair of the key and the hash of the value,
// and then appends it to simpleMap's kv pairs.
// and then appends it to SimpleMap's kv pairs.
func (sm *simpleMap) Set(key string, value []byte) {
sm.sorted = false
@ -128,7 +128,7 @@ func (sm *simpleMap) Set(key string, value []byte) {
// and make a determination to fetch or not.
vhash := tmhash.Sum(value)
sm.kvs = append(sm.kvs, kv.Pair{
sm.Kvs = append(sm.Kvs, kv.Pair{
Key: []byte(key),
Value: vhash,
})
@ -138,14 +138,14 @@ func (sm *simpleMap) Set(key string, value []byte) {
// (UNSTABLE: and by value too if duplicate key).
func (sm *simpleMap) Hash() []byte {
sm.Sort()
return hashKVPairs(sm.kvs)
return hashKVPairs(sm.Kvs)
}
func (sm *simpleMap) Sort() {
if sm.sorted {
return
}
sm.kvs.Sort()
sm.Kvs.Sort()
sm.sorted = true
}
@ -153,8 +153,8 @@ func (sm *simpleMap) Sort() {
// NOTE these contain the hashed key and value.
func (sm *simpleMap) KVPairs() kv.Pairs {
sm.Sort()
kvs := make(kv.Pairs, len(sm.kvs))
copy(kvs, sm.kvs)
kvs := make(kv.Pairs, len(sm.Kvs))
copy(kvs, sm.Kvs)
return kvs
}
@ -188,3 +188,41 @@ func (kv KVPair) Bytes() []byte {
}
return b.Bytes()
}
// SimpleHashFromMap computes a merkle tree from sorted map and returns the merkle
// root.
func SimpleHashFromMap(m map[string][]byte) []byte {
mm := newMerkleMap()
for k, v := range m {
mm.set(k, v)
}
return mm.hash()
}
// SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values
// in the underlying key-value pairs.
// The keys are sorted before the proofs are computed.
func SimpleProofsFromMap(m map[string][]byte) ([]byte, map[string]*merkle.SimpleProof, []string) {
sm := newSimpleMap()
for k, v := range m {
sm.Set(k, v)
}
sm.Sort()
kvs := sm.Kvs
kvsBytes := make([][]byte, len(kvs))
for i, kvp := range kvs {
kvsBytes[i] = KVPair(kvp).Bytes()
}
rootHash, proofList := merkle.SimpleProofsFromByteSlices(kvsBytes)
proofs := make(map[string]*merkle.SimpleProof)
keys := make([]string, len(proofList))
for i, kvp := range kvs {
proofs[string(kvp.Key)] = proofList[i]
keys[i] = string(kvp.Key)
}
return rootHash, proofs, keys
}

View File

@ -1,4 +1,4 @@
package rootmulti
package maps
import (
"fmt"

View File

@ -0,0 +1,98 @@
package proofs
import (
"fmt"
"math/bits"
ics23 "github.com/confio/ics23/go"
"github.com/tendermint/tendermint/crypto/merkle"
)
// ConvertExistenceProof will convert the given proof into a valid
// existence proof, if that's what it is.
//
// This is the simplest case of the range proof and we will focus on
// demoing compatibility here
func ConvertExistenceProof(p *merkle.SimpleProof, key, value []byte) (*ics23.ExistenceProof, error) {
path, err := convertInnerOps(p)
if err != nil {
return nil, err
}
proof := &ics23.ExistenceProof{
Key: key,
Value: value,
Leaf: convertLeafOp(),
Path: path,
}
return proof, nil
}
// this is adapted from merkle/hash.go:leafHash()
// and merkle/simple_map.go:KVPair.Bytes()
func convertLeafOp() *ics23.LeafOp {
prefix := []byte{0}
return &ics23.LeafOp{
Hash: ics23.HashOp_SHA256,
PrehashKey: ics23.HashOp_NO_HASH,
PrehashValue: ics23.HashOp_SHA256,
Length: ics23.LengthOp_VAR_PROTO,
Prefix: prefix,
}
}
func convertInnerOps(p *merkle.SimpleProof) ([]*ics23.InnerOp, error) {
inners := make([]*ics23.InnerOp, 0, len(p.Aunts))
path := buildPath(p.Index, p.Total)
if len(p.Aunts) != len(path) {
return nil, fmt.Errorf("calculated a path different length (%d) than provided by SimpleProof (%d)", len(path), len(p.Aunts))
}
for i, aunt := range p.Aunts {
auntRight := path[i]
// combine with: 0x01 || lefthash || righthash
inner := &ics23.InnerOp{Hash: ics23.HashOp_SHA256}
if auntRight {
inner.Prefix = []byte{1}
inner.Suffix = aunt
} else {
inner.Prefix = append([]byte{1}, aunt...)
}
inners = append(inners, inner)
}
return inners, nil
}
// buildPath returns a list of steps from leaf to root
// in each step, true means index is left side, false index is right side
// code adapted from merkle/simple_proof.go:computeHashFromAunts
func buildPath(idx int, total int) []bool {
if total < 2 {
return nil
}
numLeft := getSplitPoint(total)
goLeft := idx < numLeft
// we put goLeft at the end of the array, as we recurse from top to bottom,
// and want the leaf to be first in array, root last
if goLeft {
return append(buildPath(idx, numLeft), goLeft)
}
return append(buildPath(idx-numLeft, total-numLeft), goLeft)
}
func getSplitPoint(length int) int {
if length < 1 {
panic("Trying to split a tree with size < 1")
}
uLength := uint(length)
bitlen := bits.Len(uLength)
k := 1 << uint(bitlen-1)
if k == length {
k >>= 1
}
return k
}

View File

@ -0,0 +1,105 @@
package proofs
import (
"bytes"
"fmt"
"testing"
)
func TestLeafOp(t *testing.T) {
proof := GenerateRangeProof(20, Middle)
converted, err := ConvertExistenceProof(proof.Proof, proof.Key, proof.Value)
if err != nil {
t.Fatal(err)
}
leaf := converted.GetLeaf()
if leaf == nil {
t.Fatalf("Missing leaf node")
}
hash, err := leaf.Apply(converted.Key, converted.Value)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(hash, proof.Proof.LeafHash) {
t.Errorf("Calculated: %X\nExpected: %X", hash, proof.Proof.LeafHash)
}
}
func TestBuildPath(t *testing.T) {
cases := map[string]struct {
idx int
total int
expected []bool
}{
"pair left": {
idx: 0,
total: 2,
expected: []bool{true},
},
"pair right": {
idx: 1,
total: 2,
expected: []bool{false},
},
"power of 2": {
idx: 3,
total: 8,
expected: []bool{false, false, true},
},
"size of 7 right most": {
idx: 6,
total: 7,
expected: []bool{false, false},
},
"size of 6 right-left (from top)": {
idx: 4,
total: 6,
expected: []bool{true, false},
},
"size of 6 left-right-left (from top)": {
idx: 2,
total: 7,
expected: []bool{true, false, true},
},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
path := buildPath(tc.idx, tc.total)
if len(path) != len(tc.expected) {
t.Fatalf("Got %v\nExpected %v", path, tc.expected)
}
for i := range path {
if path[i] != tc.expected[i] {
t.Fatalf("Differ at %d\nGot %v\nExpected %v", i, path, tc.expected)
}
}
})
}
}
func TestConvertProof(t *testing.T) {
for i := 0; i < 100; i++ {
t.Run(fmt.Sprintf("Run %d", i), func(t *testing.T) {
proof := GenerateRangeProof(57, Left)
converted, err := ConvertExistenceProof(proof.Proof, proof.Key, proof.Value)
if err != nil {
t.Fatal(err)
}
calc, err := converted.Calculate()
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(calc, proof.RootHash) {
t.Errorf("Calculated: %X\nExpected: %X", calc, proof.RootHash)
}
})
}
}

View File

@ -0,0 +1,103 @@
package proofs
import (
"fmt"
"sort"
ics23 "github.com/confio/ics23/go"
sdkmaps "github.com/cosmos/cosmos-sdk/store/rootmulti/internal/maps"
)
// TendermintSpec constrains the format from ics23-tendermint (crypto/merkle SimpleProof)
var TendermintSpec = &ics23.ProofSpec{
LeafSpec: &ics23.LeafOp{
Prefix: []byte{0},
Hash: ics23.HashOp_SHA256,
PrehashValue: ics23.HashOp_SHA256,
Length: ics23.LengthOp_VAR_PROTO,
},
InnerSpec: &ics23.InnerSpec{
ChildOrder: []int32{0, 1},
MinPrefixLength: 1,
MaxPrefixLength: 1, // fixed prefix + one child
ChildSize: 32, // (no length byte)
Hash: ics23.HashOp_SHA256,
},
}
/*
CreateMembershipProof will produce a CommitmentProof that the given key (and queries value) exists in the iavl tree.
If the key doesn't exist in the tree, this will return an error.
*/
func CreateMembershipProof(data map[string][]byte, key []byte) (*ics23.CommitmentProof, error) {
exist, err := createExistenceProof(data, key)
if err != nil {
return nil, err
}
proof := &ics23.CommitmentProof{
Proof: &ics23.CommitmentProof_Exist{
Exist: exist,
},
}
return proof, nil
}
/*
CreateNonMembershipProof will produce a CommitmentProof that the given key doesn't exist in the iavl tree.
If the key exists in the tree, this will return an error.
*/
func CreateNonMembershipProof(data map[string][]byte, key []byte) (*ics23.CommitmentProof, error) {
// ensure this key is not in the store
if _, ok := data[string(key)]; ok {
return nil, fmt.Errorf("cannot create non-membership proof if key is in map")
}
keys := SortedKeys(data)
rightidx := sort.SearchStrings(keys, string(key))
var err error
nonexist := &ics23.NonExistenceProof{
Key: key,
}
// include left proof unless key is left of entire map
if rightidx >= 1 {
leftkey := keys[rightidx-1]
nonexist.Left, err = createExistenceProof(data, []byte(leftkey))
if err != nil {
return nil, err
}
}
// include right proof unless key is right of entire map
if rightidx < len(keys) {
rightkey := keys[rightidx]
nonexist.Right, err = createExistenceProof(data, []byte(rightkey))
if err != nil {
return nil, err
}
}
proof := &ics23.CommitmentProof{
Proof: &ics23.CommitmentProof_Nonexist{
Nonexist: nonexist,
},
}
return proof, nil
}
func createExistenceProof(data map[string][]byte, key []byte) (*ics23.ExistenceProof, error) {
value, ok := data[string(key)]
if !ok {
return nil, fmt.Errorf("cannot make existence proof if key is not in map")
}
_, ics23, _ := sdkmaps.SimpleProofsFromMap(data)
proof := ics23[string(key)]
if proof == nil {
return nil, fmt.Errorf("returned no proof for key")
}
return ConvertExistenceProof(proof, key, value)
}

View File

@ -0,0 +1,89 @@
package proofs
import (
"testing"
ics23 "github.com/confio/ics23/go"
)
func TestCreateMembership(t *testing.T) {
cases := map[string]struct {
size int
loc Where
}{
"small left": {size: 100, loc: Left},
"small middle": {size: 100, loc: Middle},
"small right": {size: 100, loc: Right},
"big left": {size: 5431, loc: Left},
"big middle": {size: 5431, loc: Middle},
"big right": {size: 5431, loc: Right},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
data := BuildMap(tc.size)
allkeys := SortedKeys(data)
key := GetKey(allkeys, tc.loc)
val := data[key]
proof, err := CreateMembershipProof(data, []byte(key))
if err != nil {
t.Fatalf("Creating Proof: %+v", err)
}
if proof.GetExist() == nil {
t.Fatal("Unexpected proof format")
}
root := CalcRoot(data)
err = proof.GetExist().Verify(TendermintSpec, root, []byte(key), val)
if err != nil {
t.Fatalf("Verifying Proof: %+v", err)
}
valid := ics23.VerifyMembership(TendermintSpec, root, proof, []byte(key), val)
if !valid {
t.Fatalf("Membership Proof Invalid")
}
})
}
}
func TestCreateNonMembership(t *testing.T) {
cases := map[string]struct {
size int
loc Where
}{
"small left": {size: 100, loc: Left},
"small middle": {size: 100, loc: Middle},
"small right": {size: 100, loc: Right},
"big left": {size: 5431, loc: Left},
"big middle": {size: 5431, loc: Middle},
"big right": {size: 5431, loc: Right},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
data := BuildMap(tc.size)
allkeys := SortedKeys(data)
key := GetNonKey(allkeys, tc.loc)
proof, err := CreateNonMembershipProof(data, []byte(key))
if err != nil {
t.Fatalf("Creating Proof: %+v", err)
}
if proof.GetNonexist() == nil {
t.Fatal("Unexpected proof format")
}
root := CalcRoot(data)
err = proof.GetNonexist().Verify(TendermintSpec, root, []byte(key))
if err != nil {
t.Fatalf("Verifying Proof: %+v", err)
}
valid := ics23.VerifyNonMembership(TendermintSpec, root, proof, []byte(key))
if !valid {
t.Fatalf("Non Membership Proof Invalid")
}
})
}
}

View File

@ -0,0 +1,103 @@
package proofs
import (
"sort"
"github.com/tendermint/tendermint/crypto/merkle"
"github.com/tendermint/tendermint/libs/rand"
)
// SimpleResult contains a merkle.SimpleProof along with all data needed to build the confio/proof
type SimpleResult struct {
Key []byte
Value []byte
Proof *merkle.SimpleProof
RootHash []byte
}
// GenerateRangeProof makes a tree of size and returns a range proof for one random element
//
// returns a range proof and the root hash of the tree
func GenerateRangeProof(size int, loc Where) *SimpleResult {
data := BuildMap(size)
root, proofs, allkeys := merkle.SimpleProofsFromMap(data)
key := GetKey(allkeys, loc)
proof := proofs[key]
res := &SimpleResult{
Key: []byte(key),
Value: toValue(key),
Proof: proof,
RootHash: root,
}
return res
}
// Where selects a location for a key - Left, Right, or Middle
type Where int
const (
Left Where = iota
Right
Middle
)
func SortedKeys(data map[string][]byte) []string {
keys := make([]string, len(data))
i := 0
for k := range data {
keys[i] = k
i++
}
sort.Strings(keys)
return keys
}
func CalcRoot(data map[string][]byte) []byte {
root, _, _ := merkle.SimpleProofsFromMap(data)
return root
}
// GetKey this returns a key, on Left/Right/Middle
func GetKey(allkeys []string, loc Where) string {
if loc == Left {
return allkeys[0]
}
if loc == Right {
return allkeys[len(allkeys)-1]
}
// select a random index between 1 and allkeys-2
idx := rand.Int()%(len(allkeys)-2) + 1
return allkeys[idx]
}
// GetNonKey returns a missing key - Left of all, Right of all, or in the Middle
func GetNonKey(allkeys []string, loc Where) string {
if loc == Left {
return string([]byte{1, 1, 1, 1})
}
if loc == Right {
return string([]byte{0xff, 0xff, 0xff, 0xff})
}
// otherwise, next to an existing key (copy before mod)
key := GetKey(allkeys, loc)
key = key[:len(key)-2] + string([]byte{255, 255})
return key
}
func toValue(key string) []byte {
return []byte("value_for_" + key)
}
// BuildMap creates random key/values and stores in a map,
// returns a list of all keys in sorted order
func BuildMap(size int) map[string][]byte {
data := make(map[string][]byte)
// insert lots of info and store the bytes
for i := 0; i < size; i++ {
key := rand.Str(20)
data[key] = toValue(key)
}
return data
}

View File

@ -5,7 +5,6 @@ import (
"io"
"strings"
ics23tendermint "github.com/confio/ics23-tendermint"
ics23 "github.com/confio/ics23/go"
"github.com/pkg/errors"
iavltree "github.com/tendermint/iavl"
@ -18,6 +17,8 @@ import (
"github.com/cosmos/cosmos-sdk/store/dbadapter"
"github.com/cosmos/cosmos-sdk/store/iavl"
"github.com/cosmos/cosmos-sdk/store/mem"
sdkmaps "github.com/cosmos/cosmos-sdk/store/rootmulti/internal/maps"
sdkproofs "github.com/cosmos/cosmos-sdk/store/rootmulti/internal/proofs"
"github.com/cosmos/cosmos-sdk/store/tracekv"
"github.com/cosmos/cosmos-sdk/store/transient"
"github.com/cosmos/cosmos-sdk/store/types"
@ -613,19 +614,19 @@ func (ci commitInfo) Hash() []byte {
if len(ci.StoreInfos) == 0 {
return nil
}
rootHash, _, _ := SimpleProofsFromMap(ci.toMap())
rootHash, _, _ := sdkmaps.SimpleProofsFromMap(ci.toMap())
return rootHash
}
func (ci commitInfo) ProofOp(storeName string) merkle.ProofOp {
cmap := ci.toMap()
_, proofs, _ := SimpleProofsFromMap(cmap)
_, proofs, _ := sdkmaps.SimpleProofsFromMap(cmap)
proof := proofs[storeName]
if proof == nil {
panic(fmt.Sprintf("ProofOp for %s but not registered store name", storeName))
}
// convert merkle.SimpleProof to CommitmentProof
existProof, err := ics23tendermint.ConvertExistenceProof(proof, []byte(storeName), cmap[storeName])
existProof, err := sdkproofs.ConvertExistenceProof(proof, []byte(storeName), cmap[storeName])
if err != nil {
panic(fmt.Errorf("could not convert simple proof to existence proof: %w", err))
}
@ -782,39 +783,3 @@ func flushMetadata(db dbm.DB, version int64, cInfo commitInfo, pruneHeights []in
panic(fmt.Errorf("error on batch write %w", err))
}
}
// SimpleHashFromMap computes a merkle tree from sorted map and returns the merkle
// root.
func SimpleHashFromMap(m map[string][]byte) []byte {
mm := newMerkleMap()
for k, v := range m {
mm.set(k, v)
}
return mm.hash()
}
// SimpleProofsFromMap generates proofs from a map. The keys/values of the map will be used as the keys/values
// in the underlying key-value pairs.
// The keys are sorted before the proofs are computed.
func SimpleProofsFromMap(m map[string][]byte) (rootHash []byte, proofs map[string]*merkle.SimpleProof, keys []string) {
sm := newSimpleMap()
for k, v := range m {
sm.Set(k, v)
}
sm.Sort()
kvs := sm.kvs
kvsBytes := make([][]byte, len(kvs))
for i, kvp := range kvs {
kvsBytes[i] = KVPair(kvp).Bytes()
}
rootHash, proofList := merkle.SimpleProofsFromByteSlices(kvsBytes)
proofs = make(map[string]*merkle.SimpleProof)
keys = make([]string, len(proofList))
for i, kvp := range kvs {
proofs[string(kvp.Key)] = proofList[i]
keys[i] = string(kvp.Key)
}
return
}

View File

@ -11,6 +11,7 @@ import (
"github.com/cosmos/cosmos-sdk/store/iavl"
"github.com/cosmos/cosmos-sdk/store/types"
sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
sdkmaps "github.com/cosmos/cosmos-sdk/store/rootmulti/internal/maps"
)
func TestStoreType(t *testing.T) {
@ -593,5 +594,5 @@ func hashStores(stores map[types.StoreKey]types.CommitKVStore) []byte {
},
}.GetHash()
}
return SimpleHashFromMap(m)
return sdkmaps.SimpleHashFromMap(m)
}