2016-08-29 12:18:00 -07:00
|
|
|
// Copyright 2016 The go-ethereum Authors
|
|
|
|
// This file is part of the go-ethereum library.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Lesser General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Lesser General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Lesser General Public License
|
|
|
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2018-07-09 05:11:49 -07:00
|
|
|
"context"
|
2016-08-29 12:18:00 -07:00
|
|
|
"crypto"
|
2018-06-20 05:06:27 -07:00
|
|
|
"crypto/rand"
|
|
|
|
"encoding/binary"
|
2016-08-29 12:18:00 -07:00
|
|
|
"fmt"
|
|
|
|
"hash"
|
|
|
|
"io"
|
|
|
|
"sync"
|
|
|
|
|
|
|
|
"github.com/ethereum/go-ethereum/common"
|
|
|
|
"github.com/ethereum/go-ethereum/crypto/sha3"
|
2018-06-20 05:06:27 -07:00
|
|
|
"github.com/ethereum/go-ethereum/swarm/bmt"
|
2016-08-29 12:18:00 -07:00
|
|
|
)
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
const MaxPO = 16
|
|
|
|
const KeyLength = 32
|
|
|
|
|
2016-08-29 12:18:00 -07:00
|
|
|
type Hasher func() hash.Hash
|
2017-09-21 13:22:51 -07:00
|
|
|
type SwarmHasher func() SwarmHash
|
2016-08-29 12:18:00 -07:00
|
|
|
|
|
|
|
// Peer is the recorded as Source on the chunk
|
|
|
|
// should probably not be here? but network should wrap chunk object
|
|
|
|
type Peer interface{}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
type Address []byte
|
2016-08-29 12:18:00 -07:00
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (a Address) Size() uint {
|
|
|
|
return uint(len(a))
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (a Address) isEqual(y Address) bool {
|
|
|
|
return bytes.Equal(a, y)
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (a Address) bits(i, j uint) uint {
|
2016-08-29 12:18:00 -07:00
|
|
|
ii := i >> 3
|
|
|
|
jj := i & 7
|
2018-06-20 05:06:27 -07:00
|
|
|
if ii >= a.Size() {
|
2016-08-29 12:18:00 -07:00
|
|
|
return 0
|
|
|
|
}
|
|
|
|
|
|
|
|
if jj+j <= 8 {
|
2018-06-20 05:06:27 -07:00
|
|
|
return uint((a[ii] >> jj) & ((1 << j) - 1))
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
res := uint(a[ii] >> jj)
|
2016-08-29 12:18:00 -07:00
|
|
|
jj = 8 - jj
|
|
|
|
j -= jj
|
|
|
|
for j != 0 {
|
|
|
|
ii++
|
|
|
|
if j < 8 {
|
2018-06-20 05:06:27 -07:00
|
|
|
res += uint(a[ii]&((1<<j)-1)) << jj
|
2016-08-29 12:18:00 -07:00
|
|
|
return res
|
|
|
|
}
|
2018-06-20 05:06:27 -07:00
|
|
|
res += uint(a[ii]) << jj
|
2016-08-29 12:18:00 -07:00
|
|
|
jj += 8
|
|
|
|
j -= 8
|
|
|
|
}
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func Proximity(one, other []byte) (ret int) {
|
|
|
|
b := (MaxPO-1)/8 + 1
|
|
|
|
if b > len(one) {
|
|
|
|
b = len(one)
|
|
|
|
}
|
|
|
|
m := 8
|
|
|
|
for i := 0; i < b; i++ {
|
|
|
|
oxo := one[i] ^ other[i]
|
|
|
|
if i == b-1 {
|
|
|
|
m = MaxPO % 8
|
|
|
|
}
|
|
|
|
for j := 0; j < m; j++ {
|
|
|
|
if (oxo>>uint8(7-j))&0x01 != 0 {
|
|
|
|
return i*8 + j
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return MaxPO
|
|
|
|
}
|
|
|
|
|
|
|
|
func IsZeroAddr(addr Address) bool {
|
|
|
|
return len(addr) == 0 || bytes.Equal(addr, ZeroAddr)
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
var ZeroAddr = Address(common.Hash{}.Bytes())
|
2016-08-29 12:18:00 -07:00
|
|
|
|
2017-09-21 13:22:51 -07:00
|
|
|
func MakeHashFunc(hash string) SwarmHasher {
|
2016-08-29 12:18:00 -07:00
|
|
|
switch hash {
|
|
|
|
case "SHA256":
|
2017-09-21 13:22:51 -07:00
|
|
|
return func() SwarmHash { return &HashWithLength{crypto.SHA256.New()} }
|
2016-08-29 12:18:00 -07:00
|
|
|
case "SHA3":
|
2017-09-21 13:22:51 -07:00
|
|
|
return func() SwarmHash { return &HashWithLength{sha3.NewKeccak256()} }
|
|
|
|
case "BMT":
|
|
|
|
return func() SwarmHash {
|
|
|
|
hasher := sha3.NewKeccak256
|
2018-06-20 05:06:27 -07:00
|
|
|
pool := bmt.NewTreePool(hasher, bmt.SegmentCount, bmt.PoolSize)
|
2017-09-21 13:22:51 -07:00
|
|
|
return bmt.New(pool)
|
|
|
|
}
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (a Address) Hex() string {
|
|
|
|
return fmt.Sprintf("%064x", []byte(a[:]))
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (a Address) Log() string {
|
|
|
|
if len(a[:]) < 8 {
|
|
|
|
return fmt.Sprintf("%x", []byte(a[:]))
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
2018-06-20 05:06:27 -07:00
|
|
|
return fmt.Sprintf("%016x", []byte(a[:8]))
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (a Address) String() string {
|
|
|
|
return fmt.Sprintf("%064x", []byte(a)[:])
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (a Address) MarshalJSON() (out []byte, err error) {
|
|
|
|
return []byte(`"` + a.String() + `"`), nil
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (a *Address) UnmarshalJSON(value []byte) error {
|
2016-08-29 12:18:00 -07:00
|
|
|
s := string(value)
|
2018-06-20 05:06:27 -07:00
|
|
|
*a = make([]byte, 32)
|
2016-08-29 12:18:00 -07:00
|
|
|
h := common.Hex2Bytes(s[1 : len(s)-1])
|
2018-06-20 05:06:27 -07:00
|
|
|
copy(*a, h)
|
2016-08-29 12:18:00 -07:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
type AddressCollection []Address
|
|
|
|
|
|
|
|
func NewAddressCollection(l int) AddressCollection {
|
|
|
|
return make(AddressCollection, l)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c AddressCollection) Len() int {
|
|
|
|
return len(c)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c AddressCollection) Less(i, j int) bool {
|
|
|
|
return bytes.Compare(c[i], c[j]) == -1
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c AddressCollection) Swap(i, j int) {
|
|
|
|
c[i], c[j] = c[j], c[i]
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Chunk also serves as a request object passed to ChunkStores
|
|
|
|
// in case it is a retrieval request, Data is nil and Size is 0
|
|
|
|
// Note that Size is not the size of the data chunk, which is Data.Size()
|
|
|
|
// but the size of the subtree encoded in the chunk
|
|
|
|
// 0 if request, to be supplied by the dpa
|
|
|
|
type Chunk struct {
|
2018-06-20 05:06:27 -07:00
|
|
|
Addr Address // always
|
|
|
|
SData []byte // nil if request, to be supplied by dpa
|
|
|
|
Size int64 // size of the data covered by the subtree encoded in this chunk
|
|
|
|
//Source Peer // peer
|
|
|
|
C chan bool // to signal data delivery by the dpa
|
|
|
|
ReqC chan bool // to signal the request done
|
|
|
|
dbStoredC chan bool // never remove a chunk from memStore before it is written to dbStore
|
|
|
|
dbStored bool
|
|
|
|
dbStoredMu *sync.Mutex
|
|
|
|
errored error // flag which is set when the chunk request has errored or timeouted
|
|
|
|
erroredMu sync.Mutex
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (c *Chunk) SetErrored(err error) {
|
|
|
|
c.erroredMu.Lock()
|
|
|
|
defer c.erroredMu.Unlock()
|
|
|
|
|
|
|
|
c.errored = err
|
|
|
|
}
|
2016-08-29 12:18:00 -07:00
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (c *Chunk) GetErrored() error {
|
|
|
|
c.erroredMu.Lock()
|
|
|
|
defer c.erroredMu.Unlock()
|
2016-08-29 12:18:00 -07:00
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
return c.errored
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewChunk(addr Address, reqC chan bool) *Chunk {
|
|
|
|
return &Chunk{
|
|
|
|
Addr: addr,
|
|
|
|
ReqC: reqC,
|
|
|
|
dbStoredC: make(chan bool),
|
|
|
|
dbStoredMu: &sync.Mutex{},
|
|
|
|
}
|
|
|
|
}
|
2016-08-29 12:18:00 -07:00
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (c *Chunk) markAsStored() {
|
|
|
|
c.dbStoredMu.Lock()
|
|
|
|
defer c.dbStoredMu.Unlock()
|
2016-08-29 12:18:00 -07:00
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
if !c.dbStored {
|
|
|
|
close(c.dbStoredC)
|
|
|
|
c.dbStored = true
|
|
|
|
}
|
|
|
|
}
|
2017-09-21 13:22:51 -07:00
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func (c *Chunk) WaitToStore() error {
|
|
|
|
<-c.dbStoredC
|
|
|
|
return c.GetErrored()
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func GenerateRandomChunk(dataSize int64) *Chunk {
|
|
|
|
return GenerateRandomChunks(dataSize, 1)[0]
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk) {
|
|
|
|
var i int
|
|
|
|
hasher := MakeHashFunc(DefaultHash)()
|
|
|
|
if dataSize > DefaultChunkSize {
|
|
|
|
dataSize = DefaultChunkSize
|
|
|
|
}
|
|
|
|
|
|
|
|
for i = 0; i < count; i++ {
|
|
|
|
chunks = append(chunks, NewChunk(nil, nil))
|
|
|
|
chunks[i].SData = make([]byte, dataSize+8)
|
|
|
|
rand.Read(chunks[i].SData)
|
|
|
|
binary.LittleEndian.PutUint64(chunks[i].SData[:8], uint64(dataSize))
|
|
|
|
hasher.ResetWithLength(chunks[i].SData[:8])
|
|
|
|
hasher.Write(chunks[i].SData[8:])
|
|
|
|
chunks[i].Addr = make([]byte, 32)
|
|
|
|
copy(chunks[i].Addr, hasher.Sum(nil))
|
|
|
|
}
|
|
|
|
|
|
|
|
return chunks
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Size, Seek, Read, ReadAt
|
|
|
|
type LazySectionReader interface {
|
2018-07-13 08:40:28 -07:00
|
|
|
Context() context.Context
|
|
|
|
Size(context.Context, chan bool) (int64, error)
|
2016-08-29 12:18:00 -07:00
|
|
|
io.Seeker
|
|
|
|
io.Reader
|
|
|
|
io.ReaderAt
|
|
|
|
}
|
|
|
|
|
|
|
|
type LazyTestSectionReader struct {
|
|
|
|
*io.SectionReader
|
|
|
|
}
|
|
|
|
|
2018-07-13 08:40:28 -07:00
|
|
|
func (r *LazyTestSectionReader) Size(context.Context, chan bool) (int64, error) {
|
2018-06-20 05:06:27 -07:00
|
|
|
return r.SectionReader.Size(), nil
|
|
|
|
}
|
|
|
|
|
2018-07-13 08:40:28 -07:00
|
|
|
func (r *LazyTestSectionReader) Context() context.Context {
|
|
|
|
return context.TODO()
|
|
|
|
}
|
|
|
|
|
2018-06-20 05:06:27 -07:00
|
|
|
type StoreParams struct {
|
|
|
|
Hash SwarmHasher `toml:"-"`
|
|
|
|
DbCapacity uint64
|
|
|
|
CacheCapacity uint
|
|
|
|
ChunkRequestsCacheCapacity uint
|
|
|
|
BaseKey []byte
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewDefaultStoreParams() *StoreParams {
|
|
|
|
return NewStoreParams(defaultLDBCapacity, defaultCacheCapacity, defaultChunkRequestsCacheCapacity, nil, nil)
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewStoreParams(ldbCap uint64, cacheCap uint, requestsCap uint, hash SwarmHasher, basekey []byte) *StoreParams {
|
|
|
|
if basekey == nil {
|
|
|
|
basekey = make([]byte, 32)
|
|
|
|
}
|
|
|
|
if hash == nil {
|
|
|
|
hash = MakeHashFunc(DefaultHash)
|
|
|
|
}
|
|
|
|
return &StoreParams{
|
|
|
|
Hash: hash,
|
|
|
|
DbCapacity: ldbCap,
|
|
|
|
CacheCapacity: cacheCap,
|
|
|
|
ChunkRequestsCacheCapacity: requestsCap,
|
|
|
|
BaseKey: basekey,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
type ChunkData []byte
|
|
|
|
|
|
|
|
type Reference []byte
|
|
|
|
|
|
|
|
// Putter is responsible to store data and create a reference for it
|
|
|
|
type Putter interface {
|
2018-07-13 08:40:28 -07:00
|
|
|
Put(context.Context, ChunkData) (Reference, error)
|
2018-06-20 05:06:27 -07:00
|
|
|
// RefSize returns the length of the Reference created by this Putter
|
|
|
|
RefSize() int64
|
|
|
|
// Close is to indicate that no more chunk data will be Put on this Putter
|
|
|
|
Close()
|
|
|
|
// Wait returns if all data has been store and the Close() was called.
|
2018-07-09 05:11:49 -07:00
|
|
|
Wait(context.Context) error
|
2018-06-20 05:06:27 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// Getter is an interface to retrieve a chunk's data by its reference
|
|
|
|
type Getter interface {
|
2018-07-13 08:40:28 -07:00
|
|
|
Get(context.Context, Reference) (ChunkData, error)
|
2018-06-20 05:06:27 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
// NOTE: this returns invalid data if chunk is encrypted
|
|
|
|
func (c ChunkData) Size() int64 {
|
|
|
|
return int64(binary.LittleEndian.Uint64(c[:8]))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c ChunkData) Data() []byte {
|
|
|
|
return c[8:]
|
|
|
|
}
|
|
|
|
|
|
|
|
type ChunkValidator interface {
|
|
|
|
Validate(addr Address, data []byte) bool
|
|
|
|
}
|
|
|
|
|
|
|
|
// Provides method for validation of content address in chunks
|
|
|
|
// Holds the corresponding hasher to create the address
|
|
|
|
type ContentAddressValidator struct {
|
|
|
|
Hasher SwarmHasher
|
|
|
|
}
|
|
|
|
|
|
|
|
// Constructor
|
|
|
|
func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator {
|
|
|
|
return &ContentAddressValidator{
|
|
|
|
Hasher: hasher,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Validate that the given key is a valid content address for the given data
|
|
|
|
func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool {
|
|
|
|
hasher := v.Hasher()
|
|
|
|
hasher.ResetWithLength(data[:8])
|
|
|
|
hasher.Write(data[8:])
|
|
|
|
hash := hasher.Sum(nil)
|
|
|
|
|
|
|
|
return bytes.Equal(hash, addr[:])
|
2016-08-29 12:18:00 -07:00
|
|
|
}
|