From c4a48902db4b25e5f063117b33c6a435228853da Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Thu, 23 May 2019 13:42:02 +0100
Subject: [PATCH 1/2] ingest: Log block and transaction hashes in display order

We should also be consistent in the SQLite database about either storing
as TEXT in display order, or storing as BLOB in wire order. But as
that's another breaking change, let's bundle it in with anything else we
need to do to address this problem.
---
 cmd/ingest/main.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/ingest/main.go b/cmd/ingest/main.go
index f4a802a..0255b68 100644
--- a/cmd/ingest/main.go
+++ b/cmd/ingest/main.go
@@ -201,7 +201,7 @@ func handleBlock(db *sql.DB, sequence int, blockData []byte) {
 	entry := log.WithFields(logrus.Fields{
 		"seqnum":        sequence,
 		"block_height":  block.GetHeight(),
-		"block_hash":    blockHash,
+		"block_hash":    hex.EncodeToString(block.GetDisplayHash()),
 		"block_version": block.GetVersion(),
 		"tx_count":      block.GetTxCount(),
 		"sapling":       block.HasSaplingTransactions(),
@@ -226,7 +226,7 @@ func handleBlock(db *sql.DB, sequence int, blockData []byte) {
 		)
 		entry = log.WithFields(logrus.Fields{
 			"block_height": block.GetHeight(),
-			"block_hash":   blockHash,
+			"block_hash":   hex.EncodeToString(block.GetDisplayHash()),
 			"tx_index":     index,
 			"tx_size":      len(tx.Bytes()),
 			"sapling":      tx.HasSaplingTransactions(),

From 49a15525233a9ccc7da2872b881b0613a757bb58 Mon Sep 17 00:00:00 2001
From: Jack Grigg <jack@z.cash>
Date: Thu, 23 May 2019 14:09:43 +0100
Subject: [PATCH 2/2] parser: Correctly serialize Equihash solution

---
 parser/block_header.go      | 54 ++++++++++++++++++++++++++++++-------
 parser/block_header_test.go |  4 +--
 2 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/parser/block_header.go b/parser/block_header.go
index 2b7181f..b606feb 100644
--- a/parser/block_header.go
+++ b/parser/block_header.go
@@ -7,13 +7,13 @@ import (
 	"log"
 	"math/big"
 
-	"github.com/zcash-hackworks/lightwalletd/parser/internal/bytestring"
 	"github.com/pkg/errors"
+	"github.com/zcash-hackworks/lightwalletd/parser/internal/bytestring"
 )
 
 const (
-	EQUIHASH_SIZE         = 1344 // size of an Equihash solution in bytes
-	SER_BLOCK_HEADER_SIZE = 1487 // size of a serialized block header
+	serBlockHeaderMinusEquihashSize = 140  // size of a serialized block header minus the Equihash solution
+	equihashSizeMainnet             = 1344 // size of a mainnet / testnet Equihash solution in bytes
 )
 
 // A block header as defined in version 2018.0-beta-29 of the Zcash Protocol Spec.
@@ -62,8 +62,47 @@ type blockHeader struct {
 	targetThreshold *big.Int
 }
 
+func CompactLengthPrefixedLen(val []byte) int {
+	length := len(val)
+	if length < 253 {
+		return 1 + length
+	} else if length < 0xffff {
+		return 1 + 2 + length
+	} else if length < 0xffff {
+		return 1 + 4 + length
+	} else {
+		return 1 + 8 + length
+	}
+}
+
+func WriteCompactLengthPrefixed(buf *bytes.Buffer, val []byte) error {
+	length := len(val)
+	if length < 253 {
+		binary.Write(buf, binary.LittleEndian, uint8(length))
+		binary.Write(buf, binary.LittleEndian, val)
+	} else if length < 0xffff {
+		binary.Write(buf, binary.LittleEndian, byte(253))
+		binary.Write(buf, binary.LittleEndian, uint16(length))
+		binary.Write(buf, binary.LittleEndian, val)
+	} else if length < 0xffff {
+		binary.Write(buf, binary.LittleEndian, byte(254))
+		binary.Write(buf, binary.LittleEndian, uint32(length))
+		binary.Write(buf, binary.LittleEndian, val)
+	} else {
+		binary.Write(buf, binary.LittleEndian, byte(255))
+		binary.Write(buf, binary.LittleEndian, uint64(length))
+		binary.Write(buf, binary.LittleEndian, val)
+	}
+	return nil
+}
+
+func (hdr *rawBlockHeader) GetSize() int {
+	return serBlockHeaderMinusEquihashSize + CompactLengthPrefixedLen(hdr.Solution)
+}
+
 func (hdr *rawBlockHeader) MarshalBinary() ([]byte, error) {
-	backing := make([]byte, 0, SER_BLOCK_HEADER_SIZE)
+	headerSize := hdr.GetSize()
+	backing := make([]byte, 0, headerSize)
 	buf := bytes.NewBuffer(backing)
 	binary.Write(buf, binary.LittleEndian, hdr.Version)
 	binary.Write(buf, binary.LittleEndian, hdr.HashPrevBlock)
@@ -72,11 +111,8 @@ func (hdr *rawBlockHeader) MarshalBinary() ([]byte, error) {
 	binary.Write(buf, binary.LittleEndian, hdr.Time)
 	binary.Write(buf, binary.LittleEndian, hdr.NBitsBytes)
 	binary.Write(buf, binary.LittleEndian, hdr.Nonce)
-	// TODO: write a Builder that knows about CompactSize
-	binary.Write(buf, binary.LittleEndian, byte(253))
-	binary.Write(buf, binary.LittleEndian, uint16(1344))
-	binary.Write(buf, binary.LittleEndian, hdr.Solution)
-	return backing[:SER_BLOCK_HEADER_SIZE], nil
+	WriteCompactLengthPrefixed(buf, hdr.Solution)
+	return backing[:headerSize], nil
 }
 
 func NewBlockHeader() *blockHeader {
diff --git a/parser/block_header_test.go b/parser/block_header_test.go
index 8a1fc5d..fadeb6e 100644
--- a/parser/block_header_test.go
+++ b/parser/block_header_test.go
@@ -91,7 +91,7 @@ func TestBlockHeader(t *testing.T) {
 		}
 		lastBlockTime = blockHeader.Time
 
-		if len(blockHeader.Solution) != EQUIHASH_SIZE {
+		if len(blockHeader.Solution) != equihashSizeMainnet {
 			t.Error("Got wrong Equihash solution size.")
 			break
 		}
@@ -103,7 +103,7 @@ func TestBlockHeader(t *testing.T) {
 			break
 		}
 
-		if !bytes.Equal(serializedHeader, blockData[:SER_BLOCK_HEADER_SIZE]) {
+		if !bytes.Equal(serializedHeader, blockData[:serBlockHeaderMinusEquihashSize+3+equihashSizeMainnet]) {
 			offset := 0
 			length := 0
 			for i := 0; i < len(serializedHeader); i++ {