storage: begin implementing sqlite3 storage
This commit is contained in:
parent
7cc7095a81
commit
0dee0b425e
1
go.mod
1
go.mod
|
@ -3,6 +3,7 @@ module github.com/gtank/ctxd
|
||||||
require (
|
require (
|
||||||
github.com/dvyukov/go-fuzz v0.0.0-20180902053217-4aff8368ef19 // indirect
|
github.com/dvyukov/go-fuzz v0.0.0-20180902053217-4aff8368ef19 // indirect
|
||||||
github.com/golang/protobuf v1.2.0
|
github.com/golang/protobuf v1.2.0
|
||||||
|
github.com/mattn/go-sqlite3 v1.10.0
|
||||||
github.com/pebbe/zmq4 v1.0.0
|
github.com/pebbe/zmq4 v1.0.0
|
||||||
github.com/pkg/errors v0.8.0
|
github.com/pkg/errors v0.8.0
|
||||||
golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3
|
golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -9,6 +9,8 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
|
||||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
|
github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
|
||||||
|
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
|
||||||
github.com/pebbe/zmq4 v1.0.0 h1:D+MSmPpqkL5PSSmnh8g51ogirUCyemThuZzLW7Nrt78=
|
github.com/pebbe/zmq4 v1.0.0 h1:D+MSmPpqkL5PSSmnh8g51ogirUCyemThuZzLW7Nrt78=
|
||||||
github.com/pebbe/zmq4 v1.0.0/go.mod h1:7N4y5R18zBiu3l0vajMUWQgZyjv464prE8RCyBcmnZM=
|
github.com/pebbe/zmq4 v1.0.0/go.mod h1:7N4y5R18zBiu3l0vajMUWQgZyjv464prE8RCyBcmnZM=
|
||||||
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
|
||||||
|
|
|
@ -35,6 +35,15 @@ func (b *block) getEncodableHash() []byte {
|
||||||
return b.hdr.getEncodableHash()
|
return b.hdr.getEncodableHash()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *block) HasSaplingTransactions() bool {
|
||||||
|
for _, tx := range b.vtx {
|
||||||
|
if tx.HasSaplingTransactions() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// GetHeight() extracts the block height from the coinbase transaction. See
|
// GetHeight() extracts the block height from the coinbase transaction. See
|
||||||
// BIP34. Returns block height on success, or -1 on error.
|
// BIP34. Returns block height on success, or -1 on error.
|
||||||
func (b *block) GetHeight() int {
|
func (b *block) GetHeight() int {
|
||||||
|
|
|
@ -6,7 +6,6 @@ import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"log"
|
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
@ -57,12 +56,12 @@ func TestCompactBlocks(t *testing.T) {
|
||||||
|
|
||||||
blockJSON, err := ioutil.ReadFile("testdata/compact_blocks.json")
|
blockJSON, err := ioutil.ReadFile("testdata/compact_blocks.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = json.Unmarshal(blockJSON, &compactTests)
|
err = json.Unmarshal(blockJSON, &compactTests)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range compactTests {
|
for _, test := range compactTests {
|
||||||
|
|
|
@ -296,6 +296,10 @@ func (tx *transaction) getEncodableHash() []byte {
|
||||||
return digest[:]
|
return digest[:]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (tx *transaction) HasSaplingTransactions() bool {
|
||||||
|
return tx.version >= 4 && (len(tx.shieldedSpends)+len(tx.shieldedOutputs)) > 0
|
||||||
|
}
|
||||||
|
|
||||||
func (tx *transaction) ToCompact(index int) *proto.CompactTx {
|
func (tx *transaction) ToCompact(index int) *proto.CompactTx {
|
||||||
ctx := &proto.CompactTx{
|
ctx := &proto.CompactTx{
|
||||||
Index: uint64(index), // index is contextual
|
Index: uint64(index), // index is contextual
|
||||||
|
|
|
@ -0,0 +1,17 @@
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import "database/sql"
|
||||||
|
|
||||||
|
func createBlockTable(conn *sql.DB) error {
|
||||||
|
tableCreation := `
|
||||||
|
CREATE TABLE IF NOT EXISTS blocks (
|
||||||
|
height INTEGER,
|
||||||
|
hash TEXT,
|
||||||
|
has_sapling_tx BOOL,
|
||||||
|
compact_encoding BLOB,
|
||||||
|
PRIMARY KEY (height, hash)
|
||||||
|
);
|
||||||
|
`
|
||||||
|
_, err := conn.Exec(tableCreation)
|
||||||
|
return err
|
||||||
|
}
|
|
@ -0,0 +1,85 @@
|
||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
protobuf "github.com/golang/protobuf/proto"
|
||||||
|
"github.com/gtank/ctxd/parser"
|
||||||
|
_ "github.com/mattn/go-sqlite3"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
conn, err := sql.Open("sqlite3", "testdata/blocks.db")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = createBlockTable(conn)
|
||||||
|
if err != nil {
|
||||||
|
conn.Close()
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ret := m.Run()
|
||||||
|
conn.Close()
|
||||||
|
//os.Remove("testdata/blocks.db")
|
||||||
|
os.Exit(ret)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFillDB(t *testing.T) {
|
||||||
|
type compactTest struct {
|
||||||
|
BlockHeight int `json:"block"`
|
||||||
|
BlockHash string `json:"hash"`
|
||||||
|
Full string `json:"full"`
|
||||||
|
Compact string `json:"compact"`
|
||||||
|
}
|
||||||
|
var compactTests []compactTest
|
||||||
|
|
||||||
|
blockJSON, err := ioutil.ReadFile("testdata/compact_blocks.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = json.Unmarshal(blockJSON, &compactTests)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
conn, err := sql.Open("sqlite3", "testdata/blocks.db")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
|
||||||
|
for _, test := range compactTests {
|
||||||
|
blockData, _ := hex.DecodeString(test.Full)
|
||||||
|
block := parser.NewBlock()
|
||||||
|
blockData, err = block.ParseFromSlice(blockData)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(errors.Wrap(err, fmt.Sprintf("parsing testnet block %d", test.BlockHeight)))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
height := block.GetHeight()
|
||||||
|
hash := hex.EncodeToString(block.GetHash())
|
||||||
|
hasSapling := block.HasSaplingTransactions()
|
||||||
|
marshaled, _ := protobuf.Marshal(block.ToCompact())
|
||||||
|
compactB64 := base64.RawStdEncoding.EncodeToString(marshaled)
|
||||||
|
|
||||||
|
insertBlock := "INSERT INTO blocks (height, hash, has_sapling_tx, compact_encoding) values (?, ?, ?, ?)"
|
||||||
|
_, err := conn.Exec(insertBlock, height, hash, hasSapling, compactB64)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(errors.Wrap(err, fmt.Sprintf("storing compact block %d", height)))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Binary file not shown.
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue