internal: create package for unsafe bytes convertion (#8733)
Co-authored-by: Alessio Treglia <alessio@tendermint.com>
This commit is contained in:
parent
0792db78b8
commit
5f2b90c3c7
|
@ -0,0 +1,2 @@
|
||||||
|
// Package conv provides internal functions for convertions and data manipulation
|
||||||
|
package conv
|
|
@ -0,0 +1,23 @@
|
||||||
|
package conv
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
// UnsafeStrToBytes uses unsafe to convert string into byte array. Returned bytes
|
||||||
|
// must not be altered after this function is called as it will cause a segmentation fault.
|
||||||
|
func UnsafeStrToBytes(s string) []byte {
|
||||||
|
var buf = *(*[]byte)(unsafe.Pointer(&s))
|
||||||
|
(*reflect.SliceHeader)(unsafe.Pointer(&buf)).Cap = len(s)
|
||||||
|
return buf
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnsafeBytesToStr is meant to make a zero allocation conversion
|
||||||
|
// from []byte -> string to speed up operations, it is not meant
|
||||||
|
// to be used generally, but for a specific pattern to delete keys
|
||||||
|
// from a map.
|
||||||
|
func UnsafeBytesToStr(b []byte) string {
|
||||||
|
hdr := (*reflect.StringHeader)(unsafe.Pointer(&b))
|
||||||
|
return *(*string)(unsafe.Pointer(hdr))
|
||||||
|
}
|
|
@ -0,0 +1,47 @@
|
||||||
|
package conv
|
||||||
|
|
||||||
|
import (
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStringSuite(t *testing.T) {
|
||||||
|
suite.Run(t, new(StringSuite))
|
||||||
|
}
|
||||||
|
|
||||||
|
type StringSuite struct{ suite.Suite }
|
||||||
|
|
||||||
|
func unsafeConvertStr() []byte {
|
||||||
|
return UnsafeStrToBytes("abc")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *StringSuite) TestUnsafeStrToBytes() {
|
||||||
|
// we convert in other function to trigger GC. We want to check that
|
||||||
|
// the underlying array in []bytes is accessible after GC will finish swapping.
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
b := unsafeConvertStr()
|
||||||
|
runtime.GC()
|
||||||
|
<-time.NewTimer(2 * time.Millisecond).C
|
||||||
|
b2 := append(b, 'd')
|
||||||
|
s.Equal("abc", string(b))
|
||||||
|
s.Equal("abcd", string(b2))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func unsafeConvertBytes() string {
|
||||||
|
return UnsafeBytesToStr([]byte("abc"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *StringSuite) TestUnsafeBytesToStr() {
|
||||||
|
// we convert in other function to trigger GC. We want to check that
|
||||||
|
// the underlying array in []bytes is accessible after GC will finish swapping.
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
str := unsafeConvertBytes()
|
||||||
|
runtime.GC()
|
||||||
|
<-time.NewTimer(2 * time.Millisecond).C
|
||||||
|
s.Equal("abc", str)
|
||||||
|
}
|
||||||
|
}
|
|
@ -4,14 +4,13 @@ import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"container/list"
|
"container/list"
|
||||||
"io"
|
"io"
|
||||||
"reflect"
|
|
||||||
"sort"
|
"sort"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
dbm "github.com/tendermint/tm-db"
|
dbm "github.com/tendermint/tm-db"
|
||||||
|
|
||||||
|
"github.com/cosmos/cosmos-sdk/internal/conv"
|
||||||
"github.com/cosmos/cosmos-sdk/store/tracekv"
|
"github.com/cosmos/cosmos-sdk/store/tracekv"
|
||||||
"github.com/cosmos/cosmos-sdk/store/types"
|
"github.com/cosmos/cosmos-sdk/store/types"
|
||||||
"github.com/cosmos/cosmos-sdk/telemetry"
|
"github.com/cosmos/cosmos-sdk/telemetry"
|
||||||
|
@ -179,35 +178,13 @@ func (store *Store) iterator(start, end []byte, ascending bool) types.Iterator {
|
||||||
return newCacheMergeIterator(parent, cache, ascending)
|
return newCacheMergeIterator(parent, cache, ascending)
|
||||||
}
|
}
|
||||||
|
|
||||||
// strToByte is meant to make a zero allocation conversion
|
|
||||||
// from string -> []byte to speed up operations, it is not meant
|
|
||||||
// to be used generally, but for a specific pattern to check for available
|
|
||||||
// keys within a domain.
|
|
||||||
func strToByte(s string) []byte {
|
|
||||||
var b []byte
|
|
||||||
hdr := (*reflect.SliceHeader)(unsafe.Pointer(&b))
|
|
||||||
hdr.Cap = len(s)
|
|
||||||
hdr.Len = len(s)
|
|
||||||
hdr.Data = (*reflect.StringHeader)(unsafe.Pointer(&s)).Data
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
// byteSliceToStr is meant to make a zero allocation conversion
|
|
||||||
// from []byte -> string to speed up operations, it is not meant
|
|
||||||
// to be used generally, but for a specific pattern to delete keys
|
|
||||||
// from a map.
|
|
||||||
func byteSliceToStr(b []byte) string {
|
|
||||||
hdr := (*reflect.StringHeader)(unsafe.Pointer(&b))
|
|
||||||
return *(*string)(unsafe.Pointer(hdr))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Constructs a slice of dirty items, to use w/ memIterator.
|
// Constructs a slice of dirty items, to use w/ memIterator.
|
||||||
func (store *Store) dirtyItems(start, end []byte) {
|
func (store *Store) dirtyItems(start, end []byte) {
|
||||||
unsorted := make([]*kv.Pair, 0)
|
unsorted := make([]*kv.Pair, 0)
|
||||||
|
|
||||||
n := len(store.unsortedCache)
|
n := len(store.unsortedCache)
|
||||||
for key := range store.unsortedCache {
|
for key := range store.unsortedCache {
|
||||||
if dbm.IsKeyInDomain(strToByte(key), start, end) {
|
if dbm.IsKeyInDomain(conv.UnsafeStrToBytes(key), start, end) {
|
||||||
cacheValue := store.cache[key]
|
cacheValue := store.cache[key]
|
||||||
unsorted = append(unsorted, &kv.Pair{Key: []byte(key), Value: cacheValue.value})
|
unsorted = append(unsorted, &kv.Pair{Key: []byte(key), Value: cacheValue.value})
|
||||||
}
|
}
|
||||||
|
@ -219,7 +196,7 @@ func (store *Store) dirtyItems(start, end []byte) {
|
||||||
}
|
}
|
||||||
} else { // Otherwise, normally delete the unsorted keys from the map.
|
} else { // Otherwise, normally delete the unsorted keys from the map.
|
||||||
for _, kv := range unsorted {
|
for _, kv := range unsorted {
|
||||||
delete(store.unsortedCache, byteSliceToStr(kv.Key))
|
delete(store.unsortedCache, conv.UnsafeBytesToStr(kv.Key))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,10 +4,9 @@ import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
|
||||||
"sort"
|
"sort"
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
|
"github.com/cosmos/cosmos-sdk/internal/conv"
|
||||||
"github.com/cosmos/cosmos-sdk/types/errors"
|
"github.com/cosmos/cosmos-sdk/types/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -21,7 +20,7 @@ type Addressable interface {
|
||||||
// Hash creates a new address from address type and key
|
// Hash creates a new address from address type and key
|
||||||
func Hash(typ string, key []byte) []byte {
|
func Hash(typ string, key []byte) []byte {
|
||||||
hasher := sha256.New()
|
hasher := sha256.New()
|
||||||
hasher.Write(unsafeStrToByteArray(typ))
|
hasher.Write(conv.UnsafeStrToBytes(typ))
|
||||||
th := hasher.Sum(nil)
|
th := hasher.Sum(nil)
|
||||||
|
|
||||||
hasher.Reset()
|
hasher.Reset()
|
||||||
|
@ -63,11 +62,3 @@ func Module(moduleName string, key []byte) []byte {
|
||||||
mKey := append([]byte(moduleName), 0)
|
mKey := append([]byte(moduleName), 0)
|
||||||
return Hash("module", append(mKey, key...))
|
return Hash("module", append(mKey, key...))
|
||||||
}
|
}
|
||||||
|
|
||||||
// unsafeStrToByteArray uses unsafe to convert string into byte array. Returned bytes
|
|
||||||
// must not be altered after this function is called as it will cause a segmentation fault.
|
|
||||||
func unsafeStrToByteArray(s string) []byte {
|
|
||||||
var buf = *(*[]byte)(unsafe.Pointer(&s))
|
|
||||||
(*reflect.SliceHeader)(unsafe.Pointer(&buf)).Cap = len(s)
|
|
||||||
return buf
|
|
||||||
}
|
|
||||||
|
|
|
@ -2,9 +2,7 @@ package address
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"runtime"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/suite"
|
"github.com/stretchr/testify/suite"
|
||||||
|
@ -77,23 +75,6 @@ func (suite *AddressSuite) TestModule() {
|
||||||
assert.NotEqual(addr2, addr3, "changing key must change address")
|
assert.NotEqual(addr2, addr3, "changing key must change address")
|
||||||
}
|
}
|
||||||
|
|
||||||
func unsafeConvertABC() []byte {
|
|
||||||
return unsafeStrToByteArray("abc")
|
|
||||||
}
|
|
||||||
|
|
||||||
func (suite *AddressSuite) TestUnsafeStrToBytes() {
|
|
||||||
// we convert in other function to trigger GC. We want to check that
|
|
||||||
// the underlying array in []bytes is accessible after GC will finish swapping.
|
|
||||||
for i := 0; i < 5; i++ {
|
|
||||||
b := unsafeConvertABC()
|
|
||||||
runtime.GC()
|
|
||||||
<-time.NewTimer(2 * time.Millisecond).C
|
|
||||||
b2 := append(b, 'd')
|
|
||||||
suite.Equal("abc", string(b))
|
|
||||||
suite.Equal("abcd", string(b2))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type addrMock struct {
|
type addrMock struct {
|
||||||
Addr []byte
|
Addr []byte
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue