add utxodb for on disk storage of transactions.

uspv/eight333.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"os"
 
+	"github.com/boltdb/bolt"
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil/bloom"
@@ -69,6 +70,10 @@ func OpenSPV(remoteNode string, hfn string,
 	s.localVersion = VERSION
 
 	// transaction store for this SPV connection
+	err = inTs.OpenDB("utxo.db")
+	if err != nil {
+		return s, err
+	}
 	s.TS = inTs
 
 	myMsgVer, err := wire.NewMsgVersionFromConn(s.con, 0, 0)
@@ -117,10 +122,31 @@ func OpenSPV(remoteNode string, hfn string,
 	go s.incomingMessageHandler()
 	s.outMsgQueue = make(chan wire.Message)
 	go s.outgoingMessageHandler()
-	s.mBlockQueue = make(chan RootAndHeight, 10) // queue of 10 requests? more?
+	s.mBlockQueue = make(chan RootAndHeight, 32) // queue depth 32 is a thing
+
 	return s, nil
 }
 
+func (ts *TxStore) OpenDB(filename string) error {
+	var err error
+	ts.StateDB, err = bolt.Open(filename, 0644, nil)
+	if err != nil {
+		return err
+	}
+	// create buckets if they're not already there
+	return ts.StateDB.Update(func(tx *bolt.Tx) error {
+		_, err = tx.CreateBucketIfNotExists(BKTUtxos)
+		if err != nil {
+			return err
+		}
+		_, err = tx.CreateBucketIfNotExists(BKTOld)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
 func (s *SPVCon) openHeaderFile(hfn string) error {
 	_, err := os.Stat(hfn)
 	if err != nil {
@@ -138,7 +164,6 @@ func (s *SPVCon) openHeaderFile(hfn string) error {
 				hfn)
 		}
 	}
-
 	s.headerFile, err = os.OpenFile(hfn, os.O_RDWR, 0600)
 	if err != nil {
 		return err

uspv/msghandler.go

@@ -43,17 +43,9 @@ func (s *SPVCon) incomingMessageHandler() {
 			fmt.Printf(" got %d txs ", len(txids))
 			//			fmt.Printf(" = got %d txs from block %s\n",
 			//				len(txids), m.Header.BlockSha().String())
-			//			var height uint32
-			// if len(txids) > 0 {
-			// make sure block is in our store before adding txs
-			//	height, err = s.HeightFromHeader(m.Header)
-			//	height = 20000
-			// if err != nil {
-			//log.Printf("Merkle block height error: %s\n", err.Error())
-			//	continue
-			//	}
-			//	}
 			rah := <-s.mBlockQueue // pop height off mblock queue
+			// this verifies order, and also that the returned header fits
+			// into our SPV header file
 			if !rah.root.IsEqual(&m.Header.MerkleRoot) {
 				log.Printf("out of order error")
 			}

uspv/txstore.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 
+	"github.com/boltdb/bolt"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
 	"github.com/btcsuite/btcutil/bloom"
@@ -13,17 +14,18 @@ import (
 type TxStore struct {
 	OKTxids map[wire.ShaHash]uint32 // known good txids and their heights
 
-	Utxos []Utxo  // stacks on stacks
-	Sum   int64   // racks on racks
-	Adrs  []MyAdr // endeavouring to acquire capital
+	Utxos   []Utxo   // stacks on stacks
+	Sum     int64    // racks on racks
+	Adrs    []MyAdr  // endeavouring to acquire capital
+	StateDB *bolt.DB // place to write all this down
 }
 
 type Utxo struct { // cash money.
 	// combo of outpoint and txout which has all the info needed to spend
-	Op       wire.OutPoint
-	Txo      wire.TxOut
-	AtHeight uint32 // block height where this tx was confirmed, 0 for unconf
-	KeyIdx   uint32 // index for private key needed to sign / spend
+	AtHeight uint32        // block height where this tx was confirmed, 0 for unconf
+	KeyIdx   uint32        // index for private key needed to sign / spend
+	Op       wire.OutPoint // where
+	Txo      wire.TxOut    // what
 }
 
 type MyAdr struct { // an address I have the private key for

uspv/utxodb.go

@@ -0,0 +1,146 @@
+package uspv
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/boltdb/bolt"
+)
+
+var (
+	BKTUtxos = []byte("DuffelBag")  // leave the rest to collect interest
+	BKTOld   = []byte("SpentTxs")   // for bookkeeping
+	KEYState = []byte("LastUpdate") // last state of DB
+)
+
+func (u *Utxo) SaveToDB(dbx *bolt.DB) error {
+	return dbx.Update(func(tx *bolt.Tx) error {
+		duf := tx.Bucket(BKTUtxos)
+		b, err := u.ToBytes()
+		if err != nil {
+			return err
+		}
+		// key : val is txid:everything else
+		return duf.Put(b[:36], b[36:])
+	})
+}
+
+func (ts *TxStore) LoadUtxos() error {
+	var kc, vc []byte
+
+	err := ts.StateDB.View(func(tx *bolt.Tx) error {
+		duf := tx.Bucket(BKTUtxos)
+		if duf == nil {
+			return fmt.Errorf("no duffel bag")
+		}
+		spent := tx.Bucket(BKTOld)
+		if spent == nil {
+			return fmt.Errorf("no spenttx bucket")
+		}
+
+		duf.ForEach(func(k, v []byte) error {
+			// have to copy these here, otherwise append will crash it.
+			// not quite sure why but append does weird stuff I guess.
+			copy(kc, k)
+			copy(vc, v)
+			if spent.Get(kc) == nil { // if it's not in the spent bucket
+				// create a new utxo
+				newU, err := UtxoFromBytes(append(kc, vc...))
+				if err != nil {
+					return err
+				}
+				// and add it to ram
+				ts.Utxos = append(ts.Utxos, newU)
+			}
+			return nil
+		})
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// ToBytes turns a Utxo into some bytes.
+// note that the txid is the first 36 bytes and in our use cases will be stripped
+// off, but is left here for other applications
+func (u *Utxo) ToBytes() ([]byte, error) {
+	var buf bytes.Buffer
+	// write 32 byte txid of the utxo
+	_, err := buf.Write(u.Op.Hash.Bytes())
+	if err != nil {
+		return nil, err
+	}
+	// write 4 byte outpoint index within the tx to spend
+	err = binary.Write(&buf, binary.BigEndian, u.Op.Index)
+	if err != nil {
+		return nil, err
+	}
+	// write 4 byte height of utxo
+	err = binary.Write(&buf, binary.BigEndian, u.AtHeight)
+	if err != nil {
+		return nil, err
+	}
+	// write 4 byte key index of utxo
+	err = binary.Write(&buf, binary.BigEndian, u.KeyIdx)
+	if err != nil {
+		return nil, err
+	}
+	// write 8 byte amount of money at the utxo
+	err = binary.Write(&buf, binary.BigEndian, u.Txo.Value)
+	if err != nil {
+		return nil, err
+	}
+
+	// write variable length (usually like 25 byte) pkscript
+	_, err = buf.Write(u.Txo.PkScript)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+// UtxoFromBytes turns bytes into a Utxo.  Note it wants the txid and outindex
+// in the first 36 bytes, which isn't stored that way in the boldDB,
+// but can be easily appended.
+func UtxoFromBytes(b []byte) (Utxo, error) {
+	var u Utxo
+	if b == nil {
+		return u, fmt.Errorf("nil input slice")
+	}
+	buf := bytes.NewBuffer(b)
+	if buf.Len() < 52 { // minimum 52 bytes with no pkscript
+		return u, fmt.Errorf("Got %d bytes for sender, expect > 52", buf.Len())
+	}
+	// read 32 byte txid
+	err := u.Op.Hash.SetBytes(buf.Next(32))
+	if err != nil {
+		return u, err
+	}
+	// read 4 byte outpoint index within the tx to spend
+	err = binary.Read(buf, binary.BigEndian, &u.Op.Index)
+	if err != nil {
+		return u, err
+	}
+	// read 4 byte height of utxo
+	err = binary.Read(buf, binary.BigEndian, &u.AtHeight)
+	if err != nil {
+		return u, err
+	}
+	// read 4 byte key index of utxo
+	err = binary.Read(buf, binary.BigEndian, &u.KeyIdx)
+	if err != nil {
+		return u, err
+	}
+	// read 8 byte amount of money at the utxo
+	err = binary.Read(buf, binary.BigEndian, &u.Txo.Value)
+	if err != nil {
+		return u, err
+	}
+	// read variable length (usually like 25 byte) pkscript
+	u.Txo.PkScript = buf.Bytes()
+	return u, nil
+}