parser: give up on doing anything clever

parser/block.go

@@ -5,6 +5,7 @@ import (
 	"crypto/sha256"
 	"encoding/binary"
 	"log"
+	"math/big"
 
 	"github.com/gtank/ctxd/parser/internal/bytestring"
 	"github.com/pkg/errors"
@@ -25,18 +26,18 @@ type rawBlockHeader struct {
 	// A SHA-256d hash in internal byte order of the previous block's header. This
 	// ensures no previous block can be changed without also changing this block's
 	// header.
-	HashPrevBlock [32]byte
+	HashPrevBlock []byte
 
 	// A SHA-256d hash in internal byte order. The merkle root is derived from
 	// the hashes of all transactions included in this block, ensuring that
 	// none of those transactions can be modified without modifying the header.
-	HashMerkleRoot [32]byte
+	HashMerkleRoot []byte
 
 	// [Pre-Sapling] A reserved field which should be ignored.
 	// [Sapling onward] The root LEBS2OSP_256(rt) of the Sapling note
 	// commitment tree corresponding to the final Sapling treestate of this
 	// block.
-	HashFinalSaplingRoot [32]byte
+	HashFinalSaplingRoot []byte
 
 	// The block time is a Unix epoch time (UTC) when the miner started hashing
 	// the header (according to the miner).
@@ -44,74 +45,112 @@ type rawBlockHeader struct {
 
 	// An encoded version of the target threshold this block's header hash must
 	// be less than or equal to, in the same nBits format used by Bitcoin.
-	NBits [4]byte
+	NBitsBytes []byte
 
 	// An arbitrary field that miners can change to modify the header hash in
 	// order to produce a hash less than or equal to the target threshold.
-	Nonce [32]byte
+	Nonce []byte
 
-	// The size of an Equihash solution in bytes (always 1344).
+	// The Equihash solution. In the wire format, this is a
-	SolutionSize EquihashSize
+	// CompactSize-prefixed value.
-
+	Solution []byte
-	// The Equihash solution.
-	Solution [EQUIHASH_SIZE]byte
 }
 
-// EquihashSize is a concrete instance of Bitcoin's CompactSize encoding. This
+type blockHeader struct {
-// representation is a hack allowing us to use Go's binary parsing. In contexts
+	*rawBlockHeader
-// outside of Zcash this could be a variable-length field.
+	cachedHash      []byte
-type EquihashSize struct {
+	targetThreshold *big.Int
-	SizeTag byte   // always the byte value 253
-	Size    uint16 // always 1344
 }
 
 func (hdr *rawBlockHeader) MarshalBinary() ([]byte, error) {
-	serBytes := make([]byte, 0, SER_BLOCK_HEADER_SIZE)
+	backing := make([]byte, 0, SER_BLOCK_HEADER_SIZE)
-	serBuf := bytes.NewBuffer(serBytes)
+	buf := bytes.NewBuffer(backing)
-	err := binary.Write(serBuf, binary.LittleEndian, hdr)
+	binary.Write(buf, binary.LittleEndian, hdr.Version)
-	return serBytes[:SER_BLOCK_HEADER_SIZE], err
+	binary.Write(buf, binary.LittleEndian, hdr.HashPrevBlock)
+	binary.Write(buf, binary.LittleEndian, hdr.HashMerkleRoot)
+	binary.Write(buf, binary.LittleEndian, hdr.HashFinalSaplingRoot)
+	binary.Write(buf, binary.LittleEndian, hdr.Time)
+	binary.Write(buf, binary.LittleEndian, hdr.NBitsBytes)
+	binary.Write(buf, binary.LittleEndian, hdr.Nonce)
+	// TODO: write a Builder that knows about CompactSize
+	binary.Write(buf, binary.LittleEndian, byte(253))
+	binary.Write(buf, binary.LittleEndian, uint16(1344))
+	binary.Write(buf, binary.LittleEndian, hdr.Solution)
+	return backing[:SER_BLOCK_HEADER_SIZE], nil
 }
 
-func (hdr *rawBlockHeader) UnmarshalBinary(data []byte) error {
+func newBlockHeader() *blockHeader {
-	reader := bytes.NewReader(data)
+	return &blockHeader{
-	err := binary.Read(reader, binary.LittleEndian, hdr)
+		rawBlockHeader: new(rawBlockHeader),
-	if err != nil {
-		return errors.Wrap(err, "failed parsing block header")
 	}
-	return nil
 }
 
-type blockHeaderDecoder struct {
+// ParseFromSlice parses the block header struct from the provided byte slice,
-	in *bytestring.String
+// advancing over the bytes read. If successful it returns the rest of the
-}
+// slice, otherwise it returns the input slice unaltered along with an error.
+func (hdr *blockHeader) ParseFromSlice(in []byte) (rest []byte, err error) {
+	s := bytestring.String(in)
 
-func NewBlockHeaderDecoder(in *bytestring.String) Decoder {
+	// Primary parsing layer: sort the bytes into things
-	return &blockHeaderDecoder{in}
-}
 
-func (dec *blockHeaderDecoder) Decode(out Serializable) error {
+	if ok := s.ReadInt32(&hdr.Version); !ok {
-	hdr, ok := out.(*BlockHeader)
+		return in, errors.New("could not read header version")
-	if !ok {
-		return errors.New("unexpected Serializable for BlockHeader decoder")
 	}
 
-	if hdr.rawBlockHeader == nil {
+	if ok := s.ReadBytes(&hdr.HashPrevBlock, 32); !ok {
-		hdr.rawBlockHeader = new(rawBlockHeader)
+		return in, errors.New("could not read HashPrevBlock")
 	}
 
-	err := binary.Read(dec.in, binary.LittleEndian, hdr.rawBlockHeader)
+	if ok := s.ReadBytes(&hdr.HashMerkleRoot, 32); !ok {
-	if err != nil {
+		return in, errors.New("could not read HashMerkleRoot")
-		return errors.Wrap(err, "parsing block header")
 	}
-	return nil
+
+	if ok := s.ReadBytes(&hdr.HashFinalSaplingRoot, 32); !ok {
+		return in, errors.New("could not read HashFinalSaplingRoot")
+	}
+
+	if ok := s.ReadUint32(&hdr.Time); !ok {
+		return in, errors.New("could not read timestamp")
+	}
+
+	if ok := s.ReadBytes(&hdr.NBitsBytes, 4); !ok {
+		return in, errors.New("could not read NBits bytes")
+	}
+
+	if ok := s.ReadBytes(&hdr.Nonce, 32); !ok {
+		return in, errors.New("could not read Nonce bytes")
+	}
+
+	if ok := s.ReadCompactLengthPrefixed((*bytestring.String)(&hdr.Solution)); !ok {
+		return in, errors.New("could not read CompactSize-prefixed Equihash solution")
+	}
+
+	// TODO interpret the bytes
+	//hdr.targetThreshold = parseNBits(hdr.NBitsBytes)
+
+	return []byte(s), nil
 }
 
-type BlockHeader struct {
+func parseNBits(b []byte) *big.Int {
-	*rawBlockHeader
+	byteLen := int(b[0])
-	cachedHash []byte
+
+	targetBytes := make([]byte, byteLen)
+	copy(targetBytes, b[1:])
+
+	// If high bit set, return a negative result. This is in the Bitcoin Core
+	// test vectors even though Bitcoin itself will never produce or interpret
+	// a difficulty lower than zero.
+	if b[1]&0x80 != 0 {
+		targetBytes[0] &= 0x7F
+		target := new(big.Int).SetBytes(targetBytes)
+		target.Neg(target)
+		return target
+	}
+
+	return new(big.Int).SetBytes(targetBytes)
 }
 
-func (hdr *BlockHeader) GetHash() []byte {
+func (hdr *blockHeader) GetHash() []byte {
 	if hdr.cachedHash != nil {
 		return hdr.cachedHash
 	}

parser/block_test.go

@@ -4,12 +4,56 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/hex"
+	"math/big"
 	"os"
 	"testing"
-
-	"github.com/gtank/ctxd/parser/internal/bytestring"
 )
 
+// https://bitcoin.org/en/developer-reference#target-nbits
+var nbitsTests = []struct {
+	bytes  []byte
+	target string
+}{
+	{
+		[]byte{0x18, 0x1b, 0xc3, 0x30},
+		"1bc330000000000000000000000000000000000000000000",
+	},
+	{
+		[]byte{0x01, 0x00, 0x34, 0x56},
+		"00",
+	},
+	{
+		[]byte{0x01, 0x12, 0x34, 0x56},
+		"12",
+	},
+	{
+		[]byte{0x02, 0x00, 0x80, 00},
+		"80",
+	},
+	{
+		[]byte{0x05, 0x00, 0x92, 0x34},
+		"92340000",
+	},
+	{
+		[]byte{0x04, 0x92, 0x34, 0x56},
+		"-12345600",
+	},
+	{
+		[]byte{0x04, 0x12, 0x34, 0x56},
+		"12345600",
+	},
+}
+
+func TestParseNBits(t *testing.T) {
+	for i, tt := range nbitsTests {
+		target := parseNBits(tt.bytes)
+		expected, _ := new(big.Int).SetString(tt.target, 16)
+		if target.Cmp(expected) != 0 {
+			t.Errorf("NBits parsing failed case %d:\nwant: %x\nhave: %x", i, expected, target)
+		}
+	}
+}
+
 func TestBlockHeader(t *testing.T) {
 	testBlocks, err := os.Open("testdata/blocks")
 	if err != nil {
@@ -22,27 +66,19 @@ func TestBlockHeader(t *testing.T) {
 	scan := bufio.NewScanner(testBlocks)
 	for scan.Scan() {
 		blockDataHex := scan.Text()
-		decodedBlockData, err := hex.DecodeString(blockDataHex)
+		blockData, err := hex.DecodeString(blockDataHex)
 		if err != nil {
 			t.Error(err)
 			continue
 		}
 
-		s := bytestring.String(decodedBlockData)
+		blockHeader := newBlockHeader()
-		startLength := len(s)
+		_, err = blockHeader.ParseFromSlice(blockData)
-		dec := NewBlockHeaderDecoder(&s)
-
-		blockHeader := BlockHeader{}
-		err = dec.Decode(&blockHeader)
 		if err != nil {
 			t.Error(err)
 			continue
 		}
 
-		if (startLength - len(s)) != SER_BLOCK_HEADER_SIZE {
-			t.Error("did not advance underlying bytestring")
-		}
-
 		// Some basic sanity checks
 		if blockHeader.Version != 4 {
 			t.Error("Read wrong version in a test block.")
@@ -55,7 +91,7 @@ func TestBlockHeader(t *testing.T) {
 		}
 		lastBlockTime = blockHeader.Time
 
-		if blockHeader.SolutionSize.Size != 1344 {
+		if len(blockHeader.Solution) != EQUIHASH_SIZE {
 			t.Error("Got wrong Equihash solution size.")
 			break
 		}
@@ -67,18 +103,23 @@ func TestBlockHeader(t *testing.T) {
 			break
 		}
 
-		if !bytes.Equal(serializedHeader, decodedBlockData[:SER_BLOCK_HEADER_SIZE]) {
+		if !bytes.Equal(serializedHeader, blockData[:SER_BLOCK_HEADER_SIZE]) {
 			offset := 0
 			length := 0
-			for i := 0; i < SER_BLOCK_HEADER_SIZE; i++ {
+			for i := 0; i < len(serializedHeader); i++ {
-				if serializedHeader[i] != decodedBlockData[i] {
+				if serializedHeader[i] != blockData[i] {
 					if offset == 0 {
 						offset = i
 					}
 					length++
 				}
 			}
-			t.Errorf("Block header failed round-trip serialization:\nwant\n%x\ngot\n%x\nat %d", serializedHeader[offset:offset+length], decodedBlockData[offset:offset+length], offset)
+			t.Errorf(
+				"Block header failed round-trip:\ngot\n%x\nwant\n%x\nfirst diff at %d",
+				serializedHeader[offset:offset+length],
+				blockData[offset:offset+length],
+				offset,
+			)
 			break
 		}
 

parser/internal/bytestring/bytestring.go

@@ -54,6 +54,17 @@ func (s *String) Skip(n int) bool {
 	return s.read(n) != nil
 }
 
+// ReadByte reads a single byte into out and advances over it. It reports if
+// the read was successful.
+func (s *String) ReadByte(out *byte) bool {
+	v := s.read(1)
+	if v == nil {
+		return false
+	}
+	*out = v[0]
+	return true
+}
+
 // ReadBytes reads n bytes into out and advances over them. It reports if the
 // read was successful.
 func (s *String) ReadBytes(out *[]byte, n int) bool {
@@ -126,6 +137,29 @@ func (s *String) ReadCompactLengthPrefixed(out *String) bool {
 	return true
 }
 
+// ReadInt32 decodes a little-endian 32-bit value into out, treating it as
+// signed, and advances over it. It reports whether the read was successful.
+func (s *String) ReadInt32(out *int32) bool {
+	var tmp uint32
+	if ok := s.ReadUint32(&tmp); !ok {
+		return false
+	}
+
+	*out = int32(tmp)
+	return true
+}
+
+// ReadUint16 decodes a little-endian, 16-bit value into out and advances over
+// it. It reports whether the read was successful.
+func (s *String) ReadUint16(out *uint16) bool {
+	v := s.read(2)
+	if v == nil {
+		return false
+	}
+	*out = uint16(v[0]) | uint16(v[1])<<8
+	return true
+}
+
 // ReadUint32 decodes a little-endian, 32-bit value into out and advances over
 // it. It reports whether the read was successful.
 func (s *String) ReadUint32(out *uint32) bool {

parser/serializable.go

@@ -1,12 +0,0 @@
-package parser
-
-import "encoding"
-
-type Serializable interface {
-	encoding.BinaryMarshaler
-	encoding.BinaryUnmarshaler
-}
-
-type Decoder interface {
-	Decode(v Serializable) error
-}