From b25df389db3c98f4b964bd39511c199f02d07715 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Tue, 26 Dec 2017 04:40:35 -0800 Subject: [PATCH] Remove Bytes, just use []byte; Use protobuf for KVPair/KI64Pair --- Makefile | 32 +++++++++++--- common/bytes.go | 53 ----------------------- common/bytes_test.go | 65 ---------------------------- common/kvpair.go | 13 ++++-- common/types.pb.go | 101 +++++++++++++++++++++++++++++++++++++++++++ common/types.proto | 24 ++++++++++ 6 files changed, 161 insertions(+), 127 deletions(-) delete mode 100644 common/bytes.go delete mode 100644 common/bytes_test.go create mode 100644 common/types.pb.go create mode 100644 common/types.proto diff --git a/Makefile b/Makefile index 29a3ac7d..af60f731 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,12 @@ -.PHONY: all test get_vendor_deps ensure_tools - GOTOOLS = \ github.com/Masterminds/glide \ - github.com/alecthomas/gometalinter.v2 -GOTOOLS_CHECK = glide gometalinter.v2 + github.com/alecthomas/gometalinter.v2 \ + github.com/gogo/protobuf/protoc-gen-gogo \ + github.com/gogo/protobuf/gogoproto +GOTOOLS_CHECK = glide gometalinter.v2 protoc protoc-gen-gogo +INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protobuf -all: check get_vendor_deps build test install metalinter +all: check get_vendor_deps protoc build test install metalinter check: check_tools @@ -13,6 +14,15 @@ check: check_tools ######################################## ### Build +protoc: + ## If you get the following error, + ## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory" + ## See https://stackoverflow.com/a/25518702 + protoc $(INCLUDE) --gogo_out=plugins=grpc:. common/*.proto + @echo "--> adding nolint declarations to protobuf generated files" + @awk '/package common/ { print "//nolint: gas"; print; next }1' common/types.pb.go > common/types.pb.go.new + @mv common/types.pb.go.new common/types.pb.go + build: # Nothing to build! @@ -33,6 +43,16 @@ get_tools: go get -u -v $(GOTOOLS) @gometalinter.v2 --install +get_protoc: + @# https://github.com/google/protobuf/releases + curl -L https://github.com/google/protobuf/releases/download/v3.4.1/protobuf-cpp-3.4.1.tar.gz | tar xvz && \ + cd protobuf-3.4.1 && \ + DIST_LANG=cpp ./configure && \ + make && \ + make install && \ + cd .. && \ + rm -rf protobuf-3.4.1 + update_tools: @echo "--> Updating tools" @go get -u $(GOTOOLS) @@ -93,4 +113,4 @@ metalinter_all: # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build check_tools get_tools update_tools get_vendor_deps test fmt metalinter metalinter_all +.PHONY: check protoc build check_tools get_tools get_protoc update_tools get_vendor_deps test fmt metalinter metalinter_all diff --git a/common/bytes.go b/common/bytes.go deleted file mode 100644 index d9ede98d..00000000 --- a/common/bytes.go +++ /dev/null @@ -1,53 +0,0 @@ -package common - -import ( - "encoding/hex" - "fmt" - "strings" -) - -// The main purpose of Bytes is to enable HEX-encoding for json/encoding. -type Bytes []byte - -// Marshal needed for protobuf compatibility -func (b Bytes) Marshal() ([]byte, error) { - return b, nil -} - -// Unmarshal needed for protobuf compatibility -func (b *Bytes) Unmarshal(data []byte) error { - *b = data - return nil -} - -// This is the point of Bytes. -func (b Bytes) MarshalJSON() ([]byte, error) { - s := strings.ToUpper(hex.EncodeToString(b)) - jb := make([]byte, len(s)+2) - jb[0] = '"' - copy(jb[1:], []byte(s)) - jb[1] = '"' - return jb, nil -} - -// This is the point of Bytes. -func (b *Bytes) UnmarshalJSON(data []byte) error { - if len(data) < 2 || data[0] != '"' || data[len(data)-1] != '"' { - return fmt.Errorf("Invalid hex string: %s", data) - } - bytes, err := hex.DecodeString(string(data[1 : len(data)-1])) - if err != nil { - return err - } - *b = bytes - return nil -} - -// Allow it to fulfill various interfaces in light-client, etc... -func (b Bytes) Bytes() []byte { - return b -} - -func (b Bytes) String() string { - return strings.ToUpper(hex.EncodeToString(b)) -} diff --git a/common/bytes_test.go b/common/bytes_test.go deleted file mode 100644 index 2ad0e692..00000000 --- a/common/bytes_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package common - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -// This is a trivial test for protobuf compatibility. -func TestMarshal(t *testing.T) { - b := []byte("hello world") - dataB := Bytes(b) - b2, err := dataB.Marshal() - assert.Nil(t, err) - assert.Equal(t, b, b2) - - var dataB2 Bytes - err = (&dataB2).Unmarshal(b) - assert.Nil(t, err) - assert.Equal(t, dataB, dataB2) -} - -// Test that the hex encoding works. -func TestJSONMarshal(t *testing.T) { - - type TestStruct struct { - B1 []byte - B2 Bytes - } - - cases := []struct { - input []byte - expected string - }{ - {[]byte(``), `{"B1":"","B2":""}`}, - {[]byte(``), `{"B1":"","B2":""}`}, - {[]byte(``), `{"B1":"","B2":""}`}, - } - - for i, tc := range cases { - t.Run(fmt.Sprintf("Case %d", i), func(t *testing.T) { - ts := TestStruct{B1: tc.input, B2: tc.input} - - // Test that it marshals correctly to JSON. - jsonBytes, err := json.Marshal(ts) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, string(jsonBytes), tc.expected) - - // TODO do fuzz testing to ensure that unmarshal fails - - // Test that unmarshaling works correctly. - ts2 := TestStruct{} - err = json.Unmarshal(jsonBytes, &ts2) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, ts2.B1, tc.input) - assert.Equal(t, ts2.B2, Bytes(tc.input)) - }) - } -} diff --git a/common/kvpair.go b/common/kvpair.go index 5faa534d..54c3a58c 100644 --- a/common/kvpair.go +++ b/common/kvpair.go @@ -8,10 +8,14 @@ import ( //---------------------------------------- // KVPair +/* +Defined in types.proto + type KVPair struct { - Key Bytes - Value Bytes + Key []byte + Value []byte } +*/ type KVPairs []KVPair @@ -35,10 +39,13 @@ func (kvs KVPairs) Sort() { sort.Sort(kvs) } //---------------------------------------- // KI64Pair +/* +Defined in types.proto type KI64Pair struct { - Key Bytes + Key []byte Value int64 } +*/ type KI64Pairs []KI64Pair diff --git a/common/types.pb.go b/common/types.pb.go new file mode 100644 index 00000000..047b7aee --- /dev/null +++ b/common/types.pb.go @@ -0,0 +1,101 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: common/types.proto + +/* +Package common is a generated protocol buffer package. + +It is generated from these files: + common/types.proto + +It has these top-level messages: + KVPair + KI64Pair +*/ +//nolint: gas +package common + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +// Define these here for compatibility but use tmlibs/common.KVPair. +type KVPair struct { + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` +} + +func (m *KVPair) Reset() { *m = KVPair{} } +func (m *KVPair) String() string { return proto.CompactTextString(m) } +func (*KVPair) ProtoMessage() {} +func (*KVPair) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{0} } + +func (m *KVPair) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *KVPair) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +// Define these here for compatibility but use tmlibs/common.KI64Pair. +type KI64Pair struct { + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value int64 `protobuf:"varint,2,opt,name=value,proto3" json:"value,omitempty"` +} + +func (m *KI64Pair) Reset() { *m = KI64Pair{} } +func (m *KI64Pair) String() string { return proto.CompactTextString(m) } +func (*KI64Pair) ProtoMessage() {} +func (*KI64Pair) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{1} } + +func (m *KI64Pair) GetKey() []byte { + if m != nil { + return m.Key + } + return nil +} + +func (m *KI64Pair) GetValue() int64 { + if m != nil { + return m.Value + } + return 0 +} + +func init() { + proto.RegisterType((*KVPair)(nil), "common.KVPair") + proto.RegisterType((*KI64Pair)(nil), "common.KI64Pair") +} + +func init() { proto.RegisterFile("common/types.proto", fileDescriptorTypes) } + +var fileDescriptorTypes = []byte{ + // 137 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4a, 0xce, 0xcf, 0xcd, + 0xcd, 0xcf, 0xd3, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, + 0x83, 0x88, 0x49, 0xe9, 0xa6, 0x67, 0x96, 0x64, 0x94, 0x26, 0xe9, 0x25, 0xe7, 0xe7, 0xea, 0xa7, + 0xe7, 0xa7, 0xe7, 0xeb, 0x83, 0xa5, 0x93, 0x4a, 0xd3, 0xc0, 0x3c, 0x30, 0x07, 0xcc, 0x82, 0x68, + 0x53, 0x32, 0xe0, 0x62, 0xf3, 0x0e, 0x0b, 0x48, 0xcc, 0x2c, 0x12, 0x12, 0xe0, 0x62, 0xce, 0x4e, + 0xad, 0x94, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x02, 0x31, 0x85, 0x44, 0xb8, 0x58, 0xcb, 0x12, + 0x73, 0x4a, 0x53, 0x25, 0x98, 0xc0, 0x62, 0x10, 0x8e, 0x92, 0x11, 0x17, 0x87, 0xb7, 0xa7, 0x99, + 0x09, 0x31, 0x7a, 0x98, 0xa1, 0x7a, 0x92, 0xd8, 0xc0, 0x96, 0x19, 0x03, 0x02, 0x00, 0x00, 0xff, + 0xff, 0x5c, 0xb8, 0x46, 0xc5, 0xb9, 0x00, 0x00, 0x00, +} diff --git a/common/types.proto b/common/types.proto new file mode 100644 index 00000000..94abcccc --- /dev/null +++ b/common/types.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; +package common; + +// For more information on gogo.proto, see: +// https://github.com/gogo/protobuf/blob/master/extensions.md +// NOTE: Try really hard not to use custom types, +// it's often complicated, broken, nor not worth it. +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + + +//---------------------------------------- +// Abstract types + +// Define these here for compatibility but use tmlibs/common.KVPair. +message KVPair { + bytes key = 1; + bytes value = 2; +} + +// Define these here for compatibility but use tmlibs/common.KI64Pair. +message KI64Pair { + bytes key = 1; + int64 value = 2; +}