Initial version of a quick & and not so dirty structual changes for solana structure

Still really verbose for struct implementors, will soften that probably by using a pure
reflection based diffing algo to start with.

diff/diff.go

@@ -0,0 +1,152 @@
+package diff
+
+import (
+	"fmt"
+	"reflect"
+
+	"go.uber.org/zap"
+)
+
+var diffeableInterface = reflect.TypeOf((*Diffeable)(nil)).Elem()
+
+func Diff(left interface{}, right interface{}) (removed, added interface{}) {
+	if traceEnabled {
+		zlog.Debug("checking diff between elements", zap.Stringer("left", reflectType{left}), zap.Stringer("right", reflectType{right}))
+	}
+
+	if left == nil && right == nil {
+		if traceEnabled {
+			zlog.Debug("both end are == nil, returning them as-is")
+		}
+
+		// Hopefully types will be all right using straight received values
+		return left, right
+	}
+
+	if left == nil {
+		if traceEnabled {
+			zlog.Debug("nil -> right, returning no removed and right added")
+		}
+
+		return reflect.Zero(reflect.TypeOf(right)).Interface(), right
+	}
+
+	if right == nil {
+		if traceEnabled {
+			zlog.Debug("left -> nil, returning left removed and no added")
+		}
+
+		return left, reflect.Zero(reflect.TypeOf(left)).Interface()
+	}
+
+	leftValue := reflect.ValueOf(left)
+	leftType := leftValue.Type()
+
+	rightValue := reflect.ValueOf(right)
+	rightType := rightValue.Type()
+	if leftType != rightType {
+		panic(fmt.Errorf("type mistmatch, left != right (type %s != type %s)", leftType, rightType))
+	}
+
+	// This is costly because it means we deeply compare at each level of check. There is probably a much better
+	// way to do this. We should implement a full walking reflection based diff instead that walks the whole thing
+	// and allocate a removed/added struct and set the field as we go. Of course, will need to be public otherwise
+	// the caller implements Diffeable and peforms the job himself. Will see, probably a good start anyway.
+	if reflect.DeepEqual(left, right) {
+		if traceEnabled {
+			zlog.Debug("left == right, returning no removed and no added")
+		}
+
+		return reflect.Zero(leftType).Interface(), reflect.Zero(rightType).Interface()
+	}
+
+	// They are the same type, so we can check either left or right to ensure that both implements diff.Diffeable
+	if leftType.Implements(diffeableInterface) {
+		if traceEnabled {
+			zlog.Debug("delegating to Diffeable to perform its job on struct")
+		}
+
+		return left.(Diffeable).Diff(right)
+	}
+
+	if leftValue.Kind() == reflect.Slice {
+		if traceEnabled {
+			zlog.Debug("performing slice compare")
+		}
+
+		return diffSlice(left, leftValue, right, rightValue)
+	}
+
+	// We know at this point that left & right are not deeply equal, not a slice and does not implement Diffeable, simply return them
+	if leftType.Comparable() {
+		return left, right
+	}
+
+	panic(fmt.Errorf("type incomparable, type %s is not a slice, nor a comparable and does not implement diff.Diffeable", leftType))
+}
+
+func diffSlice(left interface{}, leftValue reflect.Value, right interface{}, rightValue reflect.Value) (interface{}, interface{}) {
+	leftLen := leftValue.Len()
+	rightLen := rightValue.Len()
+
+	if leftLen == 0 && rightLen == 0 {
+		return nil, nil
+	}
+
+	if leftLen == 0 {
+		return reflect.Zero(leftValue.Type()).Interface(), right
+	}
+
+	if rightLen == 0 {
+		return left, reflect.Zero(rightValue.Type()).Interface()
+	}
+
+	removed := reflect.Zero(rightValue.Type())
+	added := reflect.Zero(rightValue.Type())
+
+	for i := 0; i < leftLen; i++ {
+		if i < rightLen {
+			// Both set has the same value
+			leftAt := leftValue.Index(i).Interface()
+			rightAt := rightValue.Index(i).Interface()
+
+			// FIXME: Re-use Diff(...) logic that same element gives already nothing so we avoid this...
+			if !reflect.DeepEqual(leftAt, rightAt) {
+				removedAt, addedAt := Diff(leftAt, rightAt)
+
+				if traceEnabled {
+					zlog.Debug("slice elements at index different", zap.Int("index", i), zap.Stringer("removed", reflectType{removedAt}), zap.Stringer("added", reflectType{addedAt}))
+				}
+
+				removed = reflect.Append(removed, reflect.ValueOf(removedAt))
+				added = reflect.Append(added, reflect.ValueOf(addedAt))
+			}
+		} else {
+			// Left is bigger than right, every element here has been removed from left
+			removed = reflect.Append(removed, leftValue.Index(i))
+		}
+	}
+
+	// Right is bigger than left, every element after (left len - 1) has been added from right
+	if rightLen > leftLen {
+		for i := leftLen; i < rightLen; i++ {
+			added = reflect.Append(added, rightValue.Index(i))
+		}
+	}
+
+	return removed.Interface(), added.Interface()
+}
+
+// FIXME: We could most probably get rid of the Diffeable interface and diff everything ourself, should not be hard follwing
+//        reflect.DeepEqual rules, probably not worth it just yet.
+type Diffeable interface {
+	// Diff performs the structural difference between itself (i.e. the receiver implementing the interface) which
+	// we call the "left" and a "right" element returning two new structure of the same type that contains only the
+	// difference between left and right. The first is the "removed" set (i.e. ) The left (receiver), the right (parameter) and the out (result) will be all of
+	// the same type.
+	//
+	// The implementer is responsible of validating the "right" elment's type and returning the appropiate "out".
+	//
+	// For a given struct,
+	Diff(right interface{}) (removed, added interface{})
+}

diff/diff_test.go

@@ -0,0 +1,238 @@
+package diff
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDiff(t *testing.T) {
+	type pair struct {
+		left  interface{}
+		right interface{}
+	}
+
+	type expected struct {
+		removed interface{}
+		added   interface{}
+	}
+
+	tests := []struct {
+		name     string
+		in       pair
+		expected expected
+	}{
+		// Plain
+
+		{"plain - left nil, right nil",
+			pair{nil, nil},
+			expected{removed: nil, added: nil},
+		},
+
+		{"plain - left nil, right set",
+			pair{nil, &plainStruct{}},
+			expected{removed: (*plainStruct)(nil), added: &plainStruct{}},
+		},
+
+		{"plain - left set, right nil",
+			pair{&plainStruct{}, nil},
+			expected{removed: &plainStruct{}, added: (*plainStruct)(nil)},
+		},
+
+		{"plain - equal",
+			pair{&plainStruct{}, &plainStruct{}},
+			expected{removed: (*plainStruct)(nil), added: (*plainStruct)(nil)},
+		},
+
+		{"plain - diff",
+			pair{&plainStruct{field: 1}, &plainStruct{field: 2}},
+			expected{removed: &plainStruct{field: 1}, added: &plainStruct{field: 2}},
+		},
+
+		// Slice
+
+		{"slice - equal both nil",
+			pair{[]string(nil), []string(nil)},
+			expected{
+				removed: []string(nil),
+				added:   []string(nil),
+			},
+		},
+
+		{"slice - equal both length 0",
+			pair{[]string{}, []string{}},
+			expected{
+				removed: []string(nil),
+				added:   []string(nil),
+			},
+		},
+
+		{"slice - diff both length 1",
+			pair{[]string{"a"}, []string{"b"}},
+			expected{
+				removed: []string{"a"},
+				added:   []string{"b"},
+			},
+		},
+
+		{"slice - diff both length 2 re-ordered",
+			pair{[]string{"a", "b"}, []string{"b", "a"}},
+			expected{
+				removed: []string{"a", "b"},
+				added:   []string{"b", "a"},
+			},
+		},
+
+		{"slice - diff left is longer than right, all different",
+			pair{[]string{"a", "b"}, []string{"c"}},
+			expected{
+				removed: []string{"a", "b"},
+				added:   []string{"c"},
+			},
+		},
+
+		{"slice - diff left is longer than right, some equals",
+			pair{[]string{"a", "b"}, []string{"a"}},
+			expected{
+				removed: []string{"b"},
+				added:   []string(nil),
+			},
+		},
+
+		{"slice - diff left is smaller than right, all different",
+			pair{[]string{"a"}, []string{"b", "c"}},
+			expected{
+				removed: []string{"a"},
+				added:   []string{"b", "c"},
+			},
+		},
+
+		{"slice - diff left is smaller than right, some equals",
+			pair{[]string{"a"}, []string{"a", "b"}},
+			expected{
+				removed: []string(nil),
+				added:   []string{"b"},
+			},
+		},
+
+		// Full
+
+		{"full - everything diff",
+			pair{
+				&topStruct{
+					literal: "a",
+					pointer: &plainStruct{field: 1},
+					array:   []string{"a", "b"},
+					child:   &childStruct{literal: "1", pointer: &plainStruct{field: 10}, array: []string{"1", "2"}},
+				},
+				&topStruct{
+					literal: "b",
+					pointer: &plainStruct{field: 2},
+					array:   []string{"b", "c"},
+					child:   &childStruct{literal: "2", pointer: &plainStruct{field: 20}, array: []string{"2"}},
+				},
+			},
+			expected{
+				removed: &topStruct{
+					literal: "a",
+					pointer: &plainStruct{field: 1},
+					array:   []string{"a", "b"},
+					child:   &childStruct{literal: "1", pointer: &plainStruct{field: 10}, array: []string{"1", "2"}},
+				},
+				added: &topStruct{
+					literal: "b",
+					pointer: &plainStruct{field: 2},
+					array:   []string{"b", "c"},
+					child:   &childStruct{literal: "2", pointer: &plainStruct{field: 20}, array: []string{"2"}},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			removed, added := Diff(test.in.left, test.in.right)
+			assert.Equal(t, test.expected.removed, removed, "removed set is different")
+			assert.Equal(t, test.expected.added, added, "added set is different")
+		})
+	}
+}
+
+type topStruct struct {
+	literal string
+	pointer *plainStruct
+	array   []string
+	child   *childStruct
+}
+
+func (s *topStruct) Diff(rightRaw interface{}) (interface{}, interface{}) {
+	left := s
+	right := rightRaw.(*topStruct)
+
+	removed := &topStruct{}
+	added := &topStruct{}
+	if left.literal != right.literal {
+		removed.literal = left.literal
+		added.literal = right.literal
+	}
+
+	removedPointer, addedPointer := Diff(left.pointer, right.pointer)
+	removed.pointer = removedPointer.(*plainStruct)
+	added.pointer = addedPointer.(*plainStruct)
+
+	removedArray, addedArray := Diff(left.array, right.array)
+	removed.array = removedArray.([]string)
+	added.array = addedArray.([]string)
+
+	removedChild, addedChild := Diff(left.child, right.child)
+	removed.child = removedChild.(*childStruct)
+	added.child = addedChild.(*childStruct)
+
+	return removed, added
+}
+
+type childStruct struct {
+	literal string
+	pointer *plainStruct
+	array   []string
+}
+
+func (s *childStruct) Diff(rightRaw interface{}) (interface{}, interface{}) {
+	left := s
+	right := rightRaw.(*childStruct)
+
+	removed := &childStruct{}
+	added := &childStruct{}
+	if left.literal != right.literal {
+		removed.literal = left.literal
+		added.literal = right.literal
+	}
+
+	removedPointer, addedPointer := Diff(left.pointer, right.pointer)
+	removed.pointer = removedPointer.(*plainStruct)
+	added.pointer = addedPointer.(*plainStruct)
+
+	removedArray, addedArray := Diff(left.array, right.array)
+	removed.array = removedArray.([]string)
+	added.array = addedArray.([]string)
+
+	return removed, added
+}
+
+type plainStruct struct {
+	field int
+}
+
+func (s *plainStruct) Diff(rightRaw interface{}) (interface{}, interface{}) {
+	left := s
+	right := rightRaw.(*plainStruct)
+
+	removed := &plainStruct{}
+	added := &plainStruct{}
+	if left.field != right.field {
+		removed.field = left.field
+		added.field = right.field
+	}
+
+	return removed, added
+}

diff/init_test.go

@@ -0,0 +1,7 @@
+package diff
+
+import "github.com/dfuse-io/logging"
+
+func init() {
+	logging.TestingOverride()
+}

diff/logging.go

@@ -0,0 +1,29 @@
+package diff
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/dfuse-io/logging"
+	"go.uber.org/zap"
+)
+
+var traceEnabled = logging.IsTraceEnabled("solana-go", "github.com/dfuse-io/solana-go/diff")
+var zlog = zap.NewNop()
+
+func init() {
+	logging.Register("github.com/dfuse-io/solana-go/diff", &zlog)
+}
+
+type reflectType struct {
+	in interface{}
+}
+
+func (r reflectType) String() string {
+	if r.in == nil {
+		return "<nil>"
+	}
+
+	valueOf := reflect.ValueOf(r.in)
+	return fmt.Sprintf("%s (zero? %t, value %s)", valueOf.Type(), valueOf.IsZero(), r.in)
+}

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	github.com/GeertJohan/go.rice v1.0.0
+	github.com/davecgh/go-spew v1.1.1
 	github.com/dfuse-io/binary v0.0.0-20201123150056-096380ef3e5d
 	github.com/dfuse-io/logging v0.0.0-20201110202154-26697de88c79
 	github.com/fatih/color v1.7.0

programs/serum/types.go

@@ -21,6 +21,7 @@ import (
 
 	bin "github.com/dfuse-io/binary"
 	"github.com/dfuse-io/solana-go"
+	"github.com/dfuse-io/solana-go/diff"
 	"go.uber.org/zap"
 )
 
@@ -104,11 +105,11 @@ func (o *Orderbook) Items(descending bool, f func(node *SlabLeafNode) error) err
 }
 
 var SlabFactoryImplDef = bin.NewVariantDefinition(bin.Uint32TypeIDEncoding, []bin.VariantType{
-	{"uninitialized", (*SlabUninitialized)(nil)},
-	{"inner_node", (*SlabInnerNode)(nil)},
-	{"leaf_node", (*SlabLeafNode)(nil)},
-	{"free_node", (*SlabFreeNode)(nil)},
-	{"last_free_node", (*SlabLastFreeNode)(nil)},
+	{Name: "uninitialized", Type: (*SlabUninitialized)(nil)},
+	{Name: "inner_node", Type: (*SlabInnerNode)(nil)},
+	{Name: "leaf_node", Type: (*SlabLeafNode)(nil)},
+	{Name: "free_node", Type: (*SlabFreeNode)(nil)},
+	{Name: "last_free_node", Type: (*SlabLastFreeNode)(nil)},
 })
 
 type Slab struct {
@@ -175,6 +176,43 @@ type EventQueueHeader struct {
 	SeqNum       uint64
 }
 
+// Diff implements diffing between two EventQueueHeader, at some point, I think I'll remove all this necessity
+// and code a reflection walker that check each struct element automatically.
+func (q *EventQueueHeader) Diff(rightRaw interface{}) (interface{}, interface{}) {
+	left := q
+	right := rightRaw.(*EventQueueHeader)
+
+	removed := &EventQueueHeader{}
+	added := &EventQueueHeader{}
+
+	if left.Serum != right.Serum {
+		removed.Serum = left.Serum
+		added.Serum = right.Serum
+	}
+
+	if left.AccountFlags != right.AccountFlags {
+		removed.AccountFlags = left.AccountFlags
+		added.AccountFlags = right.AccountFlags
+	}
+
+	if left.Head != right.Head {
+		removed.Head = left.Head
+		added.Head = right.Head
+	}
+
+	if left.Count != right.Count {
+		removed.Count = left.Count
+		added.Count = right.Count
+	}
+
+	if left.SeqNum != right.SeqNum {
+		removed.SeqNum = left.SeqNum
+		added.SeqNum = right.SeqNum
+	}
+
+	return removed, added
+}
+
 type EventFlag uint8
 
 const (
@@ -215,6 +253,67 @@ func (e *Event) Filled() bool {
 	return Has(uint8(e.Flag), uint8(EventFlagFill))
 }
 
+// Diff implements diffing between two Event, at some point, I think I'll remove all this necessity
+// and code a reflection walker that check each struct element automatically.
+func (q *Event) Diff(rightRaw interface{}) (interface{}, interface{}) {
+	left := q
+	right := rightRaw.(*Event)
+
+	removed := &Event{}
+	added := &Event{}
+	if left.Flag != right.Flag {
+		removed.Flag = left.Flag
+		added.Flag = right.Flag
+	}
+
+	if left.OwnerSlot != right.OwnerSlot {
+		removed.OwnerSlot = left.OwnerSlot
+		added.OwnerSlot = right.OwnerSlot
+	}
+
+	if left.FeeTier != right.FeeTier {
+		removed.FeeTier = left.FeeTier
+		added.FeeTier = right.FeeTier
+	}
+
+	if left.Padding != right.Padding {
+		removed.Padding = left.Padding
+		added.Padding = right.Padding
+	}
+
+	if left.NativeQtyReleased != right.NativeQtyReleased {
+		removed.NativeQtyReleased = left.NativeQtyReleased
+		added.NativeQtyReleased = right.NativeQtyReleased
+	}
+
+	if left.NativeQtyPaid != right.NativeQtyPaid {
+		removed.NativeQtyPaid = left.NativeQtyPaid
+		added.NativeQtyPaid = right.NativeQtyPaid
+	}
+
+	if left.NativeFeeOrRebate != right.NativeFeeOrRebate {
+		removed.NativeFeeOrRebate = left.NativeFeeOrRebate
+		added.NativeFeeOrRebate = right.NativeFeeOrRebate
+	}
+
+	if left.OrderID.Lo != right.OrderID.Lo || left.OrderID.Hi != right.OrderID.Hi {
+		removed.OrderID = left.OrderID
+		added.OrderID = right.OrderID
+	}
+
+	if left.Owner == right.Owner {
+		removed.Owner = left.Owner
+		added.Owner = right.Owner
+	}
+
+	if left.ClientOrderID != right.ClientOrderID {
+		removed.ClientOrderID = left.ClientOrderID
+		added.ClientOrderID = right.ClientOrderID
+	}
+
+	return removed, added
+}
+
 func Has(b, flag uint8) bool { return b&flag != 0 }
 
 type EventQueue struct {
@@ -251,6 +350,26 @@ func (q *EventQueue) Decode(data []byte) error {
 	return nil
 }
 
+// Diff implements diffing between two EventQueue, at some point, I think I'll remove all this necessity
+// and code a reflection walker that check each struct element automatically.
+func (q *EventQueue) Diff(rightRaw interface{}) (interface{}, interface{}) {
+	left := q
+	right := rightRaw.(*EventQueue)
+
+	removed := &EventQueue{}
+	added := &EventQueue{}
+
+	removedHeader, addedHeader := diff.Diff(left.Header, right.Header)
+	removed.Header = removedHeader.(*EventQueueHeader)
+	added.Header = addedHeader.(*EventQueueHeader)
+
+	removedEvents, addedEvents := diff.Diff(left.Events, right.Events)
+	removed.Events = removedEvents.([]*Event)
+	added.Events = addedEvents.([]*Event)
+
+	return removed, added
+}
+
 type OpenOrdersV2 struct {
 	SerumPadding           [5]byte `json:"-"`
 	AccountFlags           bin.Uint64

programs/serum/types_test.go

@@ -8,7 +8,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"testing"
+	"time"
 
+	"github.com/davecgh/go-spew/spew"
+	"github.com/dfuse-io/solana-go/diff"
 	"github.com/dfuse-io/solana-go/rpc"
 
 	bin "github.com/dfuse-io/binary"
@@ -56,6 +59,37 @@ func TestDecoder_Event(t *testing.T) {
 
 }
 
+func TestDecoder_EventQueue_Diff(t *testing.T) {
+	client := rpc.NewClient("http://api.mainnet-beta.solana.com:80/rpc")
+
+	info, err := client.GetAccountInfo(context.Background(), solana.MustPublicKeyFromBase58("13iGJcA4w5hcJZDjJbJQor1zUiDLE4jv2rMW9HkD5Eo1"))
+	require.NoError(t, err)
+
+	q1 := &EventQueue{}
+	err = q1.Decode(info.Value.Data)
+	require.NoError(t, err)
+
+	time.Sleep(900 * time.Millisecond)
+
+	info, err = client.GetAccountInfo(context.Background(), solana.MustPublicKeyFromBase58("13iGJcA4w5hcJZDjJbJQor1zUiDLE4jv2rMW9HkD5Eo1"))
+	require.NoError(t, err)
+
+	q2 := &EventQueue{}
+	err = q2.Decode(info.Value.Data)
+	require.NoError(t, err)
+
+	fmt.Println("Diffing")
+	removed, added := diff.Diff(q1, q2)
+
+	fmt.Println("======== Removed ===========")
+	spew.Dump(removed)
+	fmt.Println("==========================")
+
+	fmt.Println("======== Added ===========")
+	spew.Dump(added)
+	fmt.Println("==========================")
+}
+
 func TestDecoder_Orderbook(t *testing.T) {
 	cnt, err := ioutil.ReadFile("./testdata/orderbook.hex")
 	require.NoError(t, err)

transaction.go

@@ -6,7 +6,6 @@ import (
 	"sort"
 
 	bin "github.com/dfuse-io/binary"
-
 	"go.uber.org/zap"
 )
 
@@ -52,9 +51,9 @@ func TransactionWithInstructions(instructions []TransactionInstruction, blockHas
 	}
 
 	// Add programID to the account list
-	for programId, _ := range programIDs {
+	for programID := range programIDs {
 		accounts = append(accounts, &AccountMeta{
-			PublicKey:  MustPublicKeyFromBase58(programId),
+			PublicKey:  MustPublicKeyFromBase58(programID),
 			IsSigner:   false,
 			IsWritable: false,
 		})