prevent discarding requests when parsing fails

This commit is contained in:
Bas van Kervel 2015-07-02 15:26:55 +02:00
parent 41de1cb723
commit c2590af7fd
2 changed files with 273 additions and 26 deletions

View File

@ -10,64 +10,134 @@ import (
) )
const ( const (
READ_TIMEOUT = 15 // read timeout in seconds READ_TIMEOUT = 60 // in seconds
MAX_REQUEST_SIZE = 1024 * 1024 MAX_REQUEST_SIZE = 1024 * 1024
MAX_RESPONSE_SIZE = 1024 * 1024 MAX_RESPONSE_SIZE = 1024 * 1024
) )
var (
// No new requests in buffer
EmptyRequestQueueError = fmt.Errorf("No incoming requests")
// Next request in buffer isn't yet complete
IncompleteRequestError = fmt.Errorf("Request incomplete")
)
// Json serialization support // Json serialization support
type JsonCodec struct { type JsonCodec struct {
c net.Conn c net.Conn
d *json.Decoder reqBuffer []byte
bytesInReqBuffer int
reqLastPos int
} }
// Create new JSON coder instance // Create new JSON coder instance
func NewJsonCoder(conn net.Conn) ApiCoder { func NewJsonCoder(conn net.Conn) ApiCoder {
return &JsonCodec{ return &JsonCodec{
c: conn, c: conn,
d: json.NewDecoder(conn), reqBuffer: make([]byte, MAX_REQUEST_SIZE),
bytesInReqBuffer: 0,
reqLastPos: 0,
}
}
// Indication if the next request in the buffer is a batch request
func (self *JsonCodec) isNextBatchReq() (bool, error) {
for i := 0; i < self.bytesInReqBuffer; i++ {
switch self.reqBuffer[i] {
case 0x20, 0x09, 0x0a, 0x0d: // allow leading whitespace (JSON whitespace RFC4627)
continue
case 0x7b: // single req
return false, nil
case 0x5b: // batch req
return true, nil
default:
return false, &json.InvalidUnmarshalError{}
}
}
return false, EmptyRequestQueueError
}
// remove parsed request from buffer
func (self *JsonCodec) resetReqbuffer(pos int) {
copy(self.reqBuffer, self.reqBuffer[pos:self.bytesInReqBuffer])
self.reqLastPos = 0
self.bytesInReqBuffer -= pos
}
// parse request in buffer
func (self *JsonCodec) nextRequest() (requests []*shared.Request, isBatch bool, err error) {
if isBatch, err := self.isNextBatchReq(); err == nil {
if isBatch {
requests = make([]*shared.Request, 0)
for ; self.reqLastPos <= self.bytesInReqBuffer; self.reqLastPos++ {
if err = json.Unmarshal(self.reqBuffer[:self.reqLastPos], &requests); err == nil {
self.resetReqbuffer(self.reqLastPos)
return requests, true, nil
}
}
return nil, true, IncompleteRequestError
} else {
request := shared.Request{}
for ; self.reqLastPos <= self.bytesInReqBuffer; self.reqLastPos++ {
if err = json.Unmarshal(self.reqBuffer[:self.reqLastPos], &request); err == nil {
requests := make([]*shared.Request, 1)
requests[0] = &request
self.resetReqbuffer(self.reqLastPos)
return requests, false, nil
}
}
return nil, true, IncompleteRequestError
}
} else {
return nil, false, err
} }
} }
// Serialize obj to JSON and write it to conn // Serialize obj to JSON and write it to conn
func (self *JsonCodec) ReadRequest() (requests []*shared.Request, isBatch bool, err error) { func (self *JsonCodec) ReadRequest() (requests []*shared.Request, isBatch bool, err error) {
if self.bytesInReqBuffer != 0 {
req, batch, err := self.nextRequest()
if err == nil {
return req, batch, err
}
if err != IncompleteRequestError {
return nil, false, err
}
}
// no/incomplete request in buffer -> read more data first
deadline := time.Now().Add(READ_TIMEOUT * time.Second) deadline := time.Now().Add(READ_TIMEOUT * time.Second)
if err := self.c.SetDeadline(deadline); err != nil { if err := self.c.SetDeadline(deadline); err != nil {
return nil, false, err return nil, false, err
} }
var retErr error
for { for {
var err error n, err := self.c.Read(self.reqBuffer[self.bytesInReqBuffer:])
singleRequest := shared.Request{} if err != nil {
if err = self.d.Decode(&singleRequest); err == nil { retErr = err
requests := make([]*shared.Request, 1) break
requests[0] = &singleRequest
return requests, false, nil
} }
fmt.Printf("err %T %v\n", err) self.bytesInReqBuffer += n
if opErr, ok := err.(*net.OpError); ok { requests, isBatch, err := self.nextRequest()
if opErr.Timeout() { if err == nil {
break return requests, isBatch, nil
}
} }
requests = make([]*shared.Request, 0) if err == IncompleteRequestError || err == EmptyRequestQueueError {
if err = self.d.Decode(&requests); err == nil { continue // need more data
return requests, true, nil
} }
if opErr, ok := err.(*net.OpError); ok { retErr = err
if opErr.Timeout() { break
break
}
}
} }
self.c.Close() // timeout self.c.Close()
return nil, false, fmt.Errorf("Timeout reading request") return nil, false, retErr
} }
func (self *JsonCodec) ReadResponse() (interface{}, error) { func (self *JsonCodec) ReadResponse() (interface{}, error) {

177
rpc/codec/json_test.go Normal file
View File

@ -0,0 +1,177 @@
package codec
import (
"bytes"
"io"
"net"
"testing"
"time"
)
type jsonTestConn struct {
buffer *bytes.Buffer
}
func newJsonTestConn(data []byte) *jsonTestConn {
return &jsonTestConn{
buffer: bytes.NewBuffer(data),
}
}
func (self *jsonTestConn) Read(p []byte) (n int, err error) {
return self.buffer.Read(p)
}
func (self *jsonTestConn) Write(p []byte) (n int, err error) {
return self.buffer.Write(p)
}
func (self *jsonTestConn) Close() error {
// not implemented
return nil
}
func (self *jsonTestConn) LocalAddr() net.Addr {
// not implemented
return nil
}
func (self *jsonTestConn) RemoteAddr() net.Addr {
// not implemented
return nil
}
func (self *jsonTestConn) SetDeadline(t time.Time) error {
return nil
}
func (self *jsonTestConn) SetReadDeadline(t time.Time) error {
return nil
}
func (self *jsonTestConn) SetWriteDeadline(t time.Time) error {
return nil
}
func TestJsonDecoderWithValidRequest(t *testing.T) {
reqdata := []byte(`{"jsonrpc":"2.0","method":"modules","params":[],"id":64}`)
decoder := newJsonTestConn(reqdata)
jsonDecoder := NewJsonCoder(decoder)
requests, batch, err := jsonDecoder.ReadRequest()
if err != nil {
t.Errorf("Read valid request failed - %v", err)
}
if len(requests) != 1 {
t.Errorf("Expected to get a single request but got %d", len(requests))
}
if batch {
t.Errorf("Got batch indication while expecting single request")
}
if requests[0].Id != float64(64) {
t.Errorf("Expected req.Id == 64 but got %v", requests[0].Id)
}
if requests[0].Method != "modules" {
t.Errorf("Expected req.Method == 'modules' got '%s'", requests[0].Method)
}
}
func TestJsonDecoderWithValidBatchRequest(t *testing.T) {
reqdata := []byte(`[{"jsonrpc":"2.0","method":"modules","params":[],"id":64},
{"jsonrpc":"2.0","method":"modules","params":[],"id":64}]`)
decoder := newJsonTestConn(reqdata)
jsonDecoder := NewJsonCoder(decoder)
requests, batch, err := jsonDecoder.ReadRequest()
if err != nil {
t.Errorf("Read valid batch request failed - %v", err)
}
if len(requests) != 2 {
t.Errorf("Expected to get two requests but got %d", len(requests))
}
if !batch {
t.Errorf("Got no batch indication while expecting batch request")
}
for i := 0; i < len(requests); i++ {
if requests[i].Id != float64(64) {
t.Errorf("Expected req.Id == 64 but got %v", requests[i].Id)
}
if requests[i].Method != "modules" {
t.Errorf("Expected req.Method == 'modules' got '%s'", requests[i].Method)
}
}
}
func TestJsonDecoderWithIncompleteMessage(t *testing.T) {
reqdata := []byte(`{"jsonrpc":"2.0","method":"modules","pa`)
decoder := newJsonTestConn(reqdata)
jsonDecoder := NewJsonCoder(decoder)
requests, batch, err := jsonDecoder.ReadRequest()
if err != io.EOF {
t.Errorf("Expected to read an incomplete request err but got %v", err)
}
// remaining message
decoder.Write([]byte(`rams":[],"id":64}`))
requests, batch, err = jsonDecoder.ReadRequest()
if err != nil {
t.Errorf("Read valid request failed - %v", err)
}
if len(requests) != 1 {
t.Errorf("Expected to get a single request but got %d", len(requests))
}
if batch {
t.Errorf("Got batch indication while expecting single request")
}
if requests[0].Id != float64(64) {
t.Errorf("Expected req.Id == 64 but got %v", requests[0].Id)
}
if requests[0].Method != "modules" {
t.Errorf("Expected req.Method == 'modules' got '%s'", requests[0].Method)
}
}
func TestJsonDecoderWithInvalidIncompleteMessage(t *testing.T) {
reqdata := []byte(`{"jsonrpc":"2.0","method":"modules","pa`)
decoder := newJsonTestConn(reqdata)
jsonDecoder := NewJsonCoder(decoder)
requests, batch, err := jsonDecoder.ReadRequest()
if err != io.EOF {
t.Errorf("Expected to read an incomplete request err but got %v", err)
}
// remaining message
decoder.Write([]byte(`rams":[],"id:64"}`))
requests, batch, err = jsonDecoder.ReadRequest()
if err == nil {
t.Errorf("Expected an error but got nil")
}
if len(requests) != 0 {
t.Errorf("Expected to get no requests but got %d", len(requests))
}
if batch {
t.Errorf("Got batch indication while expecting non batch")
}
}