feat(orm): add ormfield.Codec (#10601)

* feat(orm): add ormvalue.Codec

* WIP

* WIP

* working tests

* update dep

* support more types, add docs

* comments

* address review comments

* updates

* add comment

api/buf.yaml

@@ -1,4 +1,5 @@
 version: v1
+name: buf.build/cosmos/cosmos-sdk
 breaking:
   use:
     - FILE

api/cosmos/orm/v1alpha1/orm.proto

@@ -43,8 +43,9 @@ message PrimaryKeyDescriptor {
   //   string fields support sorted iteration.
   //   - bytes are encoded as raw bytes in terminal segments and length-prefixed
   //   with a single byte in non-terminal segments. Because of this byte arrays
-  //   longer than 255 bytes cannot be used in keys and bytes fields should not
+  //   longer than 255 bytes are unsupported and bytes fields should not
-  //   be assumed to be lexically sorted.
+  //   be assumed to be lexically sorted. If you have a byte array longer than
+  //   255 bytes that you'd like to index, you should consider hashing it first.
   //   - int32, sint32, int64, sint64 are encoding as fixed width bytes with
   //   an encoding that enables sorted iteration.
   //   - google.protobuf.Timestamp and google.protobuf.Duration are encoded

buf.work.yaml

@@ -8,3 +8,4 @@ directories:
   - api
   - proto
   - third_party/proto
+  - orm/internal

orm/encoding/ormfield/bool.go

@@ -0,0 +1,54 @@
+package ormfield
+
+import (
+	io "io"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// BoolCodec encodes a bool value as a single byte 0 or 1.
+type BoolCodec struct{}
+
+func (b BoolCodec) Decode(r Reader) (protoreflect.Value, error) {
+	x, err := r.ReadByte()
+	return protoreflect.ValueOfBool(x != 0), err
+}
+
+var (
+	zeroBz = []byte{0}
+	oneBz  = []byte{1}
+)
+
+func (b BoolCodec) Encode(value protoreflect.Value, w io.Writer) error {
+	var err error
+	if value.Bool() {
+		_, err = w.Write(oneBz)
+	} else {
+		_, err = w.Write(zeroBz)
+	}
+	return err
+}
+
+func (b BoolCodec) Compare(v1, v2 protoreflect.Value) int {
+	b1 := v1.Bool()
+	b2 := v2.Bool()
+	if b1 == b2 {
+		return 0
+	} else if b1 {
+		return -1
+	} else {
+		return 1
+	}
+}
+
+func (b BoolCodec) IsOrdered() bool {
+	return false
+}
+
+func (b BoolCodec) FixedBufferSize() int {
+	return 1
+}
+
+func (b BoolCodec) ComputeBufferSize(protoreflect.Value) (int, error) {
+	return b.FixedBufferSize(), nil
+}

orm/encoding/ormfield/bytes.go

@@ -0,0 +1,99 @@
+package ormfield
+
+import (
+	"bytes"
+	"io"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+
+	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
+)
+
+// BytesCodec encodes bytes as raw bytes. It errors if the byte array is longer
+// than 255 bytes.
+type BytesCodec struct{}
+
+func (b BytesCodec) FixedBufferSize() int {
+	return -1
+}
+
+func (b BytesCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
+	return bytesSize(value)
+}
+
+func bytesSize(value protoreflect.Value) (int, error) {
+	bz := value.Bytes()
+	n := len(bz)
+	if n > 255 {
+		return -1, ormerrors.BytesFieldTooLong
+	}
+	return n, nil
+}
+
+func (b BytesCodec) IsOrdered() bool {
+	return false
+}
+
+func (b BytesCodec) Decode(r Reader) (protoreflect.Value, error) {
+	bz, err := io.ReadAll(r)
+	return protoreflect.ValueOfBytes(bz), err
+}
+
+func (b BytesCodec) Encode(value protoreflect.Value, w io.Writer) error {
+	_, err := w.Write(value.Bytes())
+	return err
+}
+
+func (b BytesCodec) Compare(v1, v2 protoreflect.Value) int {
+	return bytes.Compare(v1.Bytes(), v2.Bytes())
+}
+
+// NonTerminalBytesCodec encodes bytes as raw bytes length prefixed by a single
+// byte. It errors if the byte array is longer than 255 bytes.
+type NonTerminalBytesCodec struct{}
+
+func (b NonTerminalBytesCodec) FixedBufferSize() int {
+	return -1
+}
+
+func (b NonTerminalBytesCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
+	n, err := bytesSize(value)
+	return n + 1, err
+}
+
+func (b NonTerminalBytesCodec) IsOrdered() bool {
+	return false
+}
+
+func (b NonTerminalBytesCodec) Compare(v1, v2 protoreflect.Value) int {
+	return bytes.Compare(v1.Bytes(), v2.Bytes())
+}
+
+func (b NonTerminalBytesCodec) Decode(r Reader) (protoreflect.Value, error) {
+	n, err := r.ReadByte()
+	if err != nil {
+		return protoreflect.Value{}, err
+	}
+
+	if n == 0 {
+		return protoreflect.ValueOfBytes([]byte{}), nil
+	}
+
+	bz := make([]byte, n)
+	_, err = r.Read(bz)
+	return protoreflect.ValueOfBytes(bz), err
+}
+
+func (b NonTerminalBytesCodec) Encode(value protoreflect.Value, w io.Writer) error {
+	bz := value.Bytes()
+	n := len(bz)
+	if n > 255 {
+		return ormerrors.BytesFieldTooLong
+	}
+	_, err := w.Write([]byte{byte(n)})
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(bz)
+	return err
+}

orm/encoding/ormfield/codec.go

@@ -0,0 +1,106 @@
+package ormfield
+
+import (
+	"io"
+
+	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
+
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// Codec defines an interface for decoding and encoding values in ORM index keys.
+type Codec interface {
+
+	// Decode decodes a value in a key.
+	Decode(r Reader) (protoreflect.Value, error)
+
+	// Encode encodes a value in a key.
+	Encode(value protoreflect.Value, w io.Writer) error
+
+	// Compare compares two values of this type and should primarily be used
+	// for testing.
+	Compare(v1, v2 protoreflect.Value) int
+
+	// IsOrdered returns true if callers can always assume that this ordering
+	// is suitable for sorted iteration.
+	IsOrdered() bool
+
+	// FixedBufferSize returns a positive value if encoders should assume a
+	// fixed size buffer for encoding. Encoders will use at most this much size
+	// to encode the value.
+	FixedBufferSize() int
+
+	// ComputeBufferSize estimates the buffer size needed to encode the field.
+	// Encoders will use at most this much size to encode the value.
+	ComputeBufferSize(value protoreflect.Value) (int, error)
+}
+
+type Reader interface {
+	io.Reader
+	io.ByteReader
+}
+
+var (
+	timestampMsgType  = (&timestamppb.Timestamp{}).ProtoReflect().Type()
+	timestampFullName = timestampMsgType.Descriptor().FullName()
+	durationMsgType   = (&durationpb.Duration{}).ProtoReflect().Type()
+	durationFullName  = durationMsgType.Descriptor().FullName()
+)
+
+// GetCodec returns the Codec for the provided field if one is defined.
+// nonTerminal should be set to true if this value is being encoded as a
+// non-terminal segment of a multi-part key.
+func GetCodec(field protoreflect.FieldDescriptor, nonTerminal bool) (Codec, error) {
+	if field == nil {
+		return nil, ormerrors.UnsupportedKeyField.Wrap("nil field")
+	}
+	if field.IsList() {
+		return nil, ormerrors.UnsupportedKeyField.Wrapf("repeated field %s", field.FullName())
+	}
+
+	if field.ContainingOneof() != nil {
+		return nil, ormerrors.UnsupportedKeyField.Wrapf("oneof field %s", field.FullName())
+	}
+
+	switch field.Kind() {
+	case protoreflect.BytesKind:
+		if nonTerminal {
+			return NonTerminalBytesCodec{}, nil
+		} else {
+			return BytesCodec{}, nil
+		}
+	case protoreflect.StringKind:
+		if nonTerminal {
+			return NonTerminalStringCodec{}, nil
+		} else {
+			return StringCodec{}, nil
+		}
+	case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
+		return Uint32Codec{}, nil
+	case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
+		return Uint64Codec{}, nil
+	case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
+		return Int32Codec{}, nil
+	case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
+		return Int64Codec{}, nil
+	case protoreflect.BoolKind:
+		return BoolCodec{}, nil
+	case protoreflect.EnumKind:
+		return EnumCodec{}, nil
+	case protoreflect.MessageKind:
+		msgName := field.Message().FullName()
+		switch msgName {
+		case timestampFullName:
+			return TimestampCodec{}, nil
+		case durationFullName:
+			return DurationCodec{}, nil
+		default:
+			return nil, ormerrors.UnsupportedKeyField.Wrapf("%s of type %s", field.FullName(), msgName)
+		}
+	default:
+		return nil, ormerrors.UnsupportedKeyField.Wrapf("%s of kind %s", field.FullName(), field.Kind())
+	}
+}

orm/encoding/ormfield/codec_test.go

@@ -0,0 +1,87 @@
+package ormfield_test
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/cosmos/cosmos-sdk/orm/encoding/ormfield"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"gotest.tools/v3/assert"
+	"pgregory.net/rapid"
+
+	"github.com/cosmos/cosmos-sdk/orm/types/ormerrors"
+
+	"github.com/cosmos/cosmos-sdk/orm/internal/testutil"
+)
+
+func TestCodec(t *testing.T) {
+	for _, ks := range testutil.TestFieldSpecs {
+		testCodec(t, ks)
+	}
+}
+
+func testCodec(t *testing.T, spec testutil.TestFieldSpec) {
+	t.Run(fmt.Sprintf("%s %v", spec.FieldName, false), func(t *testing.T) {
+		testCodecNT(t, spec.FieldName, spec.Gen, false)
+	})
+	t.Run(fmt.Sprintf("%s %v", spec.FieldName, true), func(t *testing.T) {
+		testCodecNT(t, spec.FieldName, spec.Gen, true)
+	})
+}
+
+func testCodecNT(t *testing.T, fname protoreflect.Name, generator *rapid.Generator, nonTerminal bool) {
+	cdc, err := testutil.MakeTestCodec(fname, nonTerminal)
+	assert.NilError(t, err)
+	rapid.Check(t, func(t *rapid.T) {
+		x := protoreflect.ValueOf(generator.Draw(t, string(fname)))
+		bz1 := checkEncodeDecodeSize(t, x, cdc)
+		if cdc.IsOrdered() {
+			y := protoreflect.ValueOf(generator.Draw(t, fmt.Sprintf("%s 2", fname)))
+			bz2 := checkEncodeDecodeSize(t, y, cdc)
+			assert.Equal(t, cdc.Compare(x, y), bytes.Compare(bz1, bz2))
+		}
+	})
+}
+
+func checkEncodeDecodeSize(t *rapid.T, x protoreflect.Value, cdc ormfield.Codec) []byte {
+	buf := &bytes.Buffer{}
+	err := cdc.Encode(x, buf)
+	assert.NilError(t, err)
+	bz := buf.Bytes()
+	size, err := cdc.ComputeBufferSize(x)
+	assert.NilError(t, err)
+	assert.Assert(t, size >= len(bz))
+	fixedSize := cdc.FixedBufferSize()
+	if fixedSize > 0 {
+		assert.Equal(t, fixedSize, size)
+	}
+	y, err := cdc.Decode(bytes.NewReader(bz))
+	assert.NilError(t, err)
+	assert.Equal(t, 0, cdc.Compare(x, y))
+	return bz
+}
+
+func TestUnsupportedFields(t *testing.T) {
+	_, err := ormfield.GetCodec(nil, false)
+	assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
+	_, err = ormfield.GetCodec(testutil.GetTestField("repeated"), false)
+	assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
+	_, err = ormfield.GetCodec(testutil.GetTestField("map"), false)
+	assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
+	_, err = ormfield.GetCodec(testutil.GetTestField("msg"), false)
+	assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
+	_, err = ormfield.GetCodec(testutil.GetTestField("oneof"), false)
+	assert.ErrorContains(t, err, ormerrors.UnsupportedKeyField.Error())
+}
+
+func TestNTBytesTooLong(t *testing.T) {
+	cdc, err := ormfield.GetCodec(testutil.GetTestField("bz"), true)
+	assert.NilError(t, err)
+	buf := &bytes.Buffer{}
+	bz := protoreflect.ValueOfBytes(make([]byte, 256))
+	assert.ErrorContains(t, cdc.Encode(bz, buf), ormerrors.BytesFieldTooLong.Error())
+	_, err = cdc.ComputeBufferSize(bz)
+	assert.ErrorContains(t, err, ormerrors.BytesFieldTooLong.Error())
+}

orm/encoding/ormfield/duration.go

@@ -0,0 +1,69 @@
+package ormfield
+
+import (
+	io "io"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+var (
+	durationSecondsField = durationMsgType.Descriptor().Fields().ByName("seconds")
+	durationNanosField   = durationMsgType.Descriptor().Fields().ByName("nanos")
+)
+
+func getDurationSecondsAndNanos(value protoreflect.Value) (protoreflect.Value, protoreflect.Value) {
+	msg := value.Message()
+	return msg.Get(durationSecondsField), msg.Get(durationNanosField)
+}
+
+// DurationCodec encodes a google.protobuf.Duration value as 12 bytes using
+// Int64Codec for seconds followed by Int32Codec for nanos. This allows for
+// sorted iteration.
+type DurationCodec struct{}
+
+func (d DurationCodec) Decode(r Reader) (protoreflect.Value, error) {
+	seconds, err := int64Codec.Decode(r)
+	if err != nil {
+		return protoreflect.Value{}, err
+	}
+	nanos, err := int32Codec.Decode(r)
+	if err != nil {
+		return protoreflect.Value{}, err
+	}
+	msg := durationMsgType.New()
+	msg.Set(durationSecondsField, seconds)
+	msg.Set(durationNanosField, nanos)
+	return protoreflect.ValueOfMessage(msg), nil
+}
+
+func (d DurationCodec) Encode(value protoreflect.Value, w io.Writer) error {
+	seconds, nanos := getDurationSecondsAndNanos(value)
+	err := int64Codec.Encode(seconds, w)
+	if err != nil {
+		return err
+	}
+	return int32Codec.Encode(nanos, w)
+}
+
+func (d DurationCodec) Compare(v1, v2 protoreflect.Value) int {
+	s1, n1 := getDurationSecondsAndNanos(v1)
+	s2, n2 := getDurationSecondsAndNanos(v2)
+	c := compareInt(s1, s2)
+	if c != 0 {
+		return c
+	} else {
+		return compareInt(n1, n2)
+	}
+}
+
+func (d DurationCodec) IsOrdered() bool {
+	return true
+}
+
+func (d DurationCodec) FixedBufferSize() int {
+	return 12
+}
+
+func (d DurationCodec) ComputeBufferSize(protoreflect.Value) (int, error) {
+	return d.FixedBufferSize(), nil
+}

orm/encoding/ormfield/enum.go

@@ -0,0 +1,48 @@
+package ormfield
+
+import (
+	"encoding/binary"
+	io "io"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// EnumCodec encodes enum values as varints.
+type EnumCodec struct{}
+
+func (e EnumCodec) Decode(r Reader) (protoreflect.Value, error) {
+	x, err := binary.ReadVarint(r)
+	return protoreflect.ValueOfEnum(protoreflect.EnumNumber(x)), err
+}
+
+func (e EnumCodec) Encode(value protoreflect.Value, w io.Writer) error {
+	x := value.Enum()
+	buf := make([]byte, binary.MaxVarintLen32)
+	n := binary.PutVarint(buf, int64(x))
+	_, err := w.Write(buf[:n])
+	return err
+}
+
+func (e EnumCodec) Compare(v1, v2 protoreflect.Value) int {
+	x := v1.Enum()
+	y := v2.Enum()
+	if x == y {
+		return 0
+	} else if x < y {
+		return -1
+	} else {
+		return 1
+	}
+}
+
+func (e EnumCodec) IsOrdered() bool {
+	return false
+}
+
+func (e EnumCodec) FixedBufferSize() int {
+	return binary.MaxVarintLen32
+}
+
+func (e EnumCodec) ComputeBufferSize(protoreflect.Value) (int, error) {
+	return e.FixedBufferSize(), nil
+}

orm/encoding/ormfield/int32.go

@@ -0,0 +1,47 @@
+package ormfield
+
+import (
+	"encoding/binary"
+	io "io"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// Int32Codec encodes 32-bit integers as big-endian unsigned 32-bit integers
+// by adding the maximum value of int32 (2147583647) + 1 before encoding so
+// that these values can be used for ordered iteration.
+type Int32Codec struct{}
+
+var int32Codec = Int32Codec{}
+
+const int32Max = 2147483647
+const int32Offset = int32Max + 1
+
+func (i Int32Codec) Decode(r Reader) (protoreflect.Value, error) {
+	var x uint32
+	err := binary.Read(r, binary.BigEndian, &x)
+	y := int64(x) - int32Offset
+	return protoreflect.ValueOfInt32(int32(y)), err
+}
+
+func (i Int32Codec) Encode(value protoreflect.Value, w io.Writer) error {
+	x := value.Int()
+	x += int32Offset
+	return binary.Write(w, binary.BigEndian, uint32(x))
+}
+
+func (i Int32Codec) Compare(v1, v2 protoreflect.Value) int {
+	return compareInt(v1, v2)
+}
+
+func (i Int32Codec) IsOrdered() bool {
+	return true
+}
+
+func (i Int32Codec) FixedBufferSize() int {
+	return 4
+}
+
+func (i Int32Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
+	return i.FixedBufferSize(), nil
+}

orm/encoding/ormfield/int64.go

@@ -0,0 +1,69 @@
+package ormfield
+
+import (
+	"encoding/binary"
+	io "io"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// Int64Codec encodes 64-bit integers as big-endian unsigned 64-bit integers
+// by adding the maximum value of int32 (9223372036854775807) + 1 before encoding so
+// that these values can be used for ordered iteration.
+type Int64Codec struct{}
+
+var int64Codec = Int64Codec{}
+
+const int64Max = 9223372036854775807
+
+func (i Int64Codec) Decode(r Reader) (protoreflect.Value, error) {
+	var x uint64
+	err := binary.Read(r, binary.BigEndian, &x)
+	if x >= int64Max {
+		x = x - int64Max - 1
+		return protoreflect.ValueOfInt64(int64(x)), err
+	} else {
+		y := int64(x) - int64Max - 1
+		return protoreflect.ValueOfInt64(y), err
+	}
+}
+
+func (i Int64Codec) Encode(value protoreflect.Value, w io.Writer) error {
+	x := value.Int()
+	if x >= -1 {
+		y := uint64(x) + int64Max + 1
+		return binary.Write(w, binary.BigEndian, y)
+	} else {
+		x += int64Max
+		x += 1
+		return binary.Write(w, binary.BigEndian, uint64(x))
+	}
+}
+
+func (i Int64Codec) Compare(v1, v2 protoreflect.Value) int {
+	return compareInt(v1, v2)
+}
+
+func (i Int64Codec) IsOrdered() bool {
+	return true
+}
+
+func (i Int64Codec) FixedBufferSize() int {
+	return 8
+}
+
+func (i Int64Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
+	return i.FixedBufferSize(), nil
+}
+
+func compareInt(v1, v2 protoreflect.Value) int {
+	x := v1.Int()
+	y := v2.Int()
+	if x == y {
+		return 0
+	} else if x < y {
+		return -1
+	} else {
+		return 1
+	}
+}

orm/encoding/ormfield/string.go

@@ -0,0 +1,87 @@
+package ormfield
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// StringCodec encodes strings as raw bytes.
+type StringCodec struct{}
+
+func (s StringCodec) FixedBufferSize() int {
+	return -1
+}
+
+func (s StringCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
+	return len(value.Interface().(string)), nil
+}
+
+func (s StringCodec) IsOrdered() bool {
+	return true
+}
+
+func (s StringCodec) Compare(v1, v2 protoreflect.Value) int {
+	return strings.Compare(v1.Interface().(string), v2.Interface().(string))
+}
+
+func (s StringCodec) Decode(r Reader) (protoreflect.Value, error) {
+	bz, err := io.ReadAll(r)
+	return protoreflect.ValueOfString(string(bz)), err
+}
+
+func (s StringCodec) Encode(value protoreflect.Value, w io.Writer) error {
+	_, err := w.Write([]byte(value.Interface().(string)))
+	return err
+}
+
+// NonTerminalStringCodec encodes strings as null-terminated raw bytes. Null
+// values within strings will produce an error.
+type NonTerminalStringCodec struct{}
+
+func (s NonTerminalStringCodec) FixedBufferSize() int {
+	return -1
+}
+
+func (s NonTerminalStringCodec) ComputeBufferSize(value protoreflect.Value) (int, error) {
+	return len(value.Interface().(string)) + 1, nil
+}
+
+func (s NonTerminalStringCodec) IsOrdered() bool {
+	return true
+}
+
+func (s NonTerminalStringCodec) Compare(v1, v2 protoreflect.Value) int {
+	return strings.Compare(v1.Interface().(string), v2.Interface().(string))
+}
+
+func (s NonTerminalStringCodec) Decode(r Reader) (protoreflect.Value, error) {
+	var bz []byte
+	for {
+		b, err := r.ReadByte()
+		if b == 0 || err == io.EOF {
+			return protoreflect.ValueOfString(string(bz)), err
+		}
+		bz = append(bz, b)
+	}
+}
+
+func (s NonTerminalStringCodec) Encode(value protoreflect.Value, w io.Writer) error {
+	str := value.Interface().(string)
+	bz := []byte(str)
+	for _, b := range bz {
+		if b == 0 {
+			return fmt.Errorf("illegal null terminator found in index string: %s", str)
+		}
+	}
+	_, err := w.Write([]byte(str))
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(nullTerminator)
+	return err
+}
+
+var nullTerminator = []byte{0}

orm/encoding/ormfield/timestamp.go

@@ -0,0 +1,69 @@
+package ormfield
+
+import (
+	"io"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// TimestampCodec DurationCodec encodes a google.protobuf.Timestamp value as 12 bytes using
+// Int64Codec for seconds followed by Int32Codec for nanos. This allows for
+// sorted iteration.
+type TimestampCodec struct{}
+
+var (
+	timestampSecondsField = timestampMsgType.Descriptor().Fields().ByName("seconds")
+	timestampNanosField   = timestampMsgType.Descriptor().Fields().ByName("nanos")
+)
+
+func getTimestampSecondsAndNanos(value protoreflect.Value) (protoreflect.Value, protoreflect.Value) {
+	msg := value.Message()
+	return msg.Get(timestampSecondsField), msg.Get(timestampNanosField)
+}
+
+func (t TimestampCodec) Decode(r Reader) (protoreflect.Value, error) {
+	seconds, err := int64Codec.Decode(r)
+	if err != nil {
+		return protoreflect.Value{}, err
+	}
+	nanos, err := int32Codec.Decode(r)
+	if err != nil {
+		return protoreflect.Value{}, err
+	}
+	msg := timestampMsgType.New()
+	msg.Set(timestampSecondsField, seconds)
+	msg.Set(timestampNanosField, nanos)
+	return protoreflect.ValueOfMessage(msg), nil
+}
+
+func (t TimestampCodec) Encode(value protoreflect.Value, w io.Writer) error {
+	seconds, nanos := getTimestampSecondsAndNanos(value)
+	err := int64Codec.Encode(seconds, w)
+	if err != nil {
+		return err
+	}
+	return int32Codec.Encode(nanos, w)
+}
+
+func (t TimestampCodec) Compare(v1, v2 protoreflect.Value) int {
+	s1, n1 := getTimestampSecondsAndNanos(v1)
+	s2, n2 := getTimestampSecondsAndNanos(v2)
+	c := compareInt(s1, s2)
+	if c != 0 {
+		return c
+	} else {
+		return compareInt(n1, n2)
+	}
+}
+
+func (t TimestampCodec) IsOrdered() bool {
+	return true
+}
+
+func (t TimestampCodec) FixedBufferSize() int {
+	return 12
+}
+
+func (t TimestampCodec) ComputeBufferSize(protoreflect.Value) (int, error) {
+	return t.FixedBufferSize(), nil
+}

orm/encoding/ormfield/uint32.go

@@ -0,0 +1,37 @@
+package ormfield
+
+import (
+	"encoding/binary"
+	"io"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// Uint32Codec encodes uint32 values as 4-byte big-endian integers.
+type Uint32Codec struct{}
+
+func (u Uint32Codec) FixedBufferSize() int {
+	return 4
+}
+
+func (u Uint32Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
+	return u.FixedBufferSize(), nil
+}
+
+func (u Uint32Codec) IsOrdered() bool {
+	return true
+}
+
+func (u Uint32Codec) Compare(v1, v2 protoreflect.Value) int {
+	return compareUint(v1, v2)
+}
+
+func (u Uint32Codec) Decode(r Reader) (protoreflect.Value, error) {
+	var x uint32
+	err := binary.Read(r, binary.BigEndian, &x)
+	return protoreflect.ValueOfUint32(x), err
+}
+
+func (u Uint32Codec) Encode(value protoreflect.Value, w io.Writer) error {
+	return binary.Write(w, binary.BigEndian, uint32(value.Uint()))
+}

orm/encoding/ormfield/uint64.go

@@ -0,0 +1,49 @@
+package ormfield
+
+import (
+	"encoding/binary"
+	"io"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// Uint64Codec encodes uint64 values as 8-byte big-endian integers.
+type Uint64Codec struct{}
+
+func (u Uint64Codec) FixedBufferSize() int {
+	return 8
+}
+
+func (u Uint64Codec) ComputeBufferSize(protoreflect.Value) (int, error) {
+	return u.FixedBufferSize(), nil
+}
+
+func (u Uint64Codec) IsOrdered() bool {
+	return true
+}
+
+func (u Uint64Codec) Compare(v1, v2 protoreflect.Value) int {
+	return compareUint(v1, v2)
+}
+
+func (u Uint64Codec) Decode(r Reader) (protoreflect.Value, error) {
+	var x uint64
+	err := binary.Read(r, binary.BigEndian, &x)
+	return protoreflect.ValueOfUint64(x), err
+}
+
+func (u Uint64Codec) Encode(value protoreflect.Value, w io.Writer) error {
+	return binary.Write(w, binary.BigEndian, value.Uint())
+}
+
+func compareUint(v1, v2 protoreflect.Value) int {
+	x := v1.Uint()
+	y := v2.Uint()
+	if x == y {
+		return 0
+	} else if x < y {
+		return -1
+	} else {
+		return 1
+	}
+}

orm/go.mod

@@ -1,3 +1,31 @@
 module github.com/cosmos/cosmos-sdk/orm
 
 go 1.17
+
+require (
+	github.com/cosmos/cosmos-proto v1.0.0-alpha1
+	github.com/cosmos/cosmos-sdk v0.44.3
+	github.com/cosmos/cosmos-sdk/api v0.1.0-alpha1
+	google.golang.org/protobuf v1.27.1
+	gotest.tools/v3 v3.0.3
+	pgregory.net/rapid v0.4.7
+)
+
+require (
+	github.com/btcsuite/btcd v0.22.0-beta // indirect
+	github.com/gogo/protobuf v1.3.3 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sasha-s/go-deadlock v0.2.1-0.20190427202633-1595213edefa // indirect
+	github.com/tendermint/tendermint v0.34.14 // indirect
+	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect
+	golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f // indirect
+	golang.org/x/sys v0.0.0-20210903071746-97244b99971b // indirect
+	golang.org/x/text v0.3.6 // indirect
+	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
+	google.golang.org/grpc v1.40.0 // indirect
+)
+
+replace github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1

orm/internal/buf.gen.yaml

@@ -0,0 +1,11 @@
+version: v1
+managed:
+  enabled: true
+  go_package_prefix:
+    default: github.com/cosmos/cosmos-sdk/orm/internal
+    override:
+      buf.build/cosmos/cosmos-sdk: github.com/cosmos/cosmos-sdk/api
+plugins:
+  - name: go-pulsar
+    out: .
+    opt: paths=source_relative

orm/internal/buf.yaml

@@ -0,0 +1,9 @@
+version: v1
+lint:
+  use:
+    - DEFAULT
+  except:
+    - PACKAGE_VERSION_SUFFIX
+breaking:
+  ignore:
+    - testpb

orm/internal/testpb/testschema.proto

@@ -0,0 +1,82 @@
+syntax = "proto3";
+
+package testpb;
+
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/duration.proto";
+import "cosmos/orm/v1alpha1/orm.proto";
+
+option go_package = "github.com/cosmos/cosmos-sdk/orm/internal/testpb";
+
+message A {
+  option (cosmos.orm.v1alpha1.table) = {
+    id: 1;
+    primary_key: {
+      fields: "u32,u64,str"
+    }
+    index:{
+      id: 1;
+      fields:"u64,str"
+    }
+    index:{
+      id: 2;
+      fields:"str,u32"
+    }
+    index:{
+      id: 3;
+      fields:"bz,str"
+    }
+  };
+
+  // Valid key fields:
+  uint32 u32 = 1;
+  uint64 u64 = 2;
+  string str = 3;
+  bytes bz = 4;
+  google.protobuf.Timestamp ts = 5;
+  google.protobuf.Duration dur = 6;
+  int32 i32 = 7;
+  sint32 s32 = 8;
+  sfixed32 sf32 = 9;
+  int64 i64 = 10;
+  sint64 s64 = 11;
+  sfixed64 sf64 = 12;
+  fixed32 f32 = 13;
+  fixed64 f64 = 14;
+  bool b = 15;
+  Enum e = 16;
+
+  // Invalid key fields:
+  repeated uint32 repeated = 17;
+  map<string, uint32> map = 18;
+  B msg = 19;
+  oneof sum {
+    uint32 oneof = 20;
+  }
+}
+
+enum Enum {
+  ENUM_UNSPECIFIED = 0;
+  ENUM_ONE = 1;
+  ENUM_TWO = 2;
+  ENUM_FIVE = 5;
+  ENUM_NEG_THREE = -3;
+}
+
+message B {
+  option (cosmos.orm.v1alpha1.singleton) = {id: 2};
+  string x = 1;
+}
+
+message C {
+  option (cosmos.orm.v1alpha1.table) = {
+    id: 3
+    primary_key:{
+      fields:"id"
+      auto_increment: true
+    }
+  };
+
+  uint64 id = 1;
+  string x = 2;
+}

orm/internal/testutil/testutil.go

@@ -0,0 +1,116 @@
+package testutil
+
+import (
+	"fmt"
+	"strings"
+
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"google.golang.org/protobuf/types/known/timestamppb"
+	"pgregory.net/rapid"
+
+	"github.com/cosmos/cosmos-sdk/orm/encoding/ormfield"
+	"github.com/cosmos/cosmos-sdk/orm/internal/testpb"
+)
+
+// TestFieldSpec defines a test field against the testpb.A message.
+type TestFieldSpec struct {
+	FieldName protoreflect.Name
+	Gen       *rapid.Generator
+}
+
+var TestFieldSpecs = []TestFieldSpec{
+	{
+		"u32",
+		rapid.Uint32(),
+	},
+	{
+		"u64",
+		rapid.Uint64(),
+	},
+	{
+		"str",
+		rapid.String().Filter(func(x string) bool {
+			// filter out null terminators
+			return strings.IndexByte(x, 0) < 0
+		}),
+	},
+	{
+		"bz",
+		rapid.SliceOfN(rapid.Byte(), 0, 255),
+	},
+	{
+		"i32",
+		rapid.Int32(),
+	},
+	{
+		"f32",
+		rapid.Uint32(),
+	},
+	{
+		"s32",
+		rapid.Int32(),
+	},
+	{
+		"sf32",
+		rapid.Int32(),
+	},
+	{
+		"i64",
+		rapid.Int64(),
+	},
+	{
+		"f64",
+		rapid.Uint64(),
+	},
+	{
+		"s64",
+		rapid.Int64(),
+	},
+	{
+		"sf64",
+		rapid.Int64(),
+	},
+	{
+		"b",
+		rapid.Bool(),
+	},
+	{
+		"ts",
+		rapid.ArrayOf(2, rapid.Int64()).Map(func(xs [2]int64) protoreflect.Message {
+			return (&timestamppb.Timestamp{
+				Seconds: xs[0],
+				Nanos:   int32(xs[1]),
+			}).ProtoReflect()
+		}),
+	},
+	{
+		"dur",
+		rapid.ArrayOf(2, rapid.Int64()).Map(func(xs [2]int64) protoreflect.Message {
+			return (&durationpb.Duration{
+				Seconds: xs[0],
+				Nanos:   int32(xs[1]),
+			}).ProtoReflect()
+		}),
+	},
+	{
+		"e",
+		rapid.Int32().Map(func(x int32) protoreflect.EnumNumber {
+			return protoreflect.EnumNumber(x)
+		}),
+	},
+}
+
+func MakeTestCodec(fname protoreflect.Name, nonTerminal bool) (ormfield.Codec, error) {
+	field := GetTestField(fname)
+	if field == nil {
+		return nil, fmt.Errorf("can't find field %s", fname)
+	}
+	return ormfield.GetCodec(field, nonTerminal)
+}
+
+func GetTestField(fname protoreflect.Name) protoreflect.FieldDescriptor {
+	a := &testpb.A{}
+	return a.ProtoReflect().Descriptor().Fields().ByName(fname)
+}

orm/types/ormerrors/errors.go

@@ -0,0 +1,10 @@
+package ormerrors
+
+import "github.com/cosmos/cosmos-sdk/types/errors"
+
+var codespace = "orm"
+
+var (
+	UnsupportedKeyField = errors.New(codespace, 1, "unsupported key field")
+	BytesFieldTooLong   = errors.New(codespace, 2, "bytes field is longer than 255 bytes")
+)

scripts/protocgen-api.sh

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-# this script generates the new API go module using pulsar
-
-set -eo pipefail
-
-cd api
-buf generate .

scripts/protocgen2.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# this script is for generating protobuf files for the new google.golang.org/protobuf API
+
+set -eo pipefail
+
+(cd api; buf generate .)
+
+(cd orm/internal; buf generate .)